ifq.c revision 1.16 
1/*	$OpenBSD: ifq.c,v 1.16 2017/12/14 00:45:16 dlg Exp $ */
2
3/*
4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/socket.h>
22#include <sys/mbuf.h>
23#include <sys/proc.h>
24
25#include <net/if.h>
26#include <net/if_var.h>
27
28/*
29 * priq glue
30 */
31unsigned int	 priq_idx(unsigned int, const struct mbuf *);
32struct mbuf	*priq_enq(struct ifqueue *, struct mbuf *);
33struct mbuf	*priq_deq_begin(struct ifqueue *, void **);
34void		 priq_deq_commit(struct ifqueue *, struct mbuf *, void *);
35void		 priq_purge(struct ifqueue *, struct mbuf_list *);
36
37void		*priq_alloc(unsigned int, void *);
38void		 priq_free(unsigned int, void *);
39
40const struct ifq_ops priq_ops = {
41	priq_idx,
42	priq_enq,
43	priq_deq_begin,
44	priq_deq_commit,
45	priq_purge,
46	priq_alloc,
47	priq_free,
48};
49
50const struct ifq_ops * const ifq_priq_ops = &priq_ops;
51
52/*
53 * priq internal structures
54 */
55
56struct priq {
57	struct mbuf_list	 pq_lists[IFQ_NQUEUES];
58};
59
60/*
61 * ifqueue serialiser
62 */
63
64void	ifq_start_task(void *);
65void	ifq_restart_task(void *);
66void	ifq_barrier_task(void *);
67
68#define TASK_ONQUEUE 0x1
69
70void
71ifq_serialize(struct ifqueue *ifq, struct task *t)
72{
73	struct task work;
74
75	if (ISSET(t->t_flags, TASK_ONQUEUE))
76		return;
77
78	mtx_enter(&ifq->ifq_task_mtx);
79	if (!ISSET(t->t_flags, TASK_ONQUEUE)) {
80		SET(t->t_flags, TASK_ONQUEUE);
81		TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry);
82	}
83
84	if (ifq->ifq_serializer == NULL) {
85		ifq->ifq_serializer = curcpu();
86
87		while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) {
88			TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry);
89			CLR(t->t_flags, TASK_ONQUEUE);
90			work = *t; /* copy to caller to avoid races */
91
92			mtx_leave(&ifq->ifq_task_mtx);
93
94			(*work.t_func)(work.t_arg);
95
96			mtx_enter(&ifq->ifq_task_mtx);
97		}
98
99		ifq->ifq_serializer = NULL;
100	}
101	mtx_leave(&ifq->ifq_task_mtx);
102}
103
104int
105ifq_is_serialized(struct ifqueue *ifq)
106{
107	return (ifq->ifq_serializer == curcpu());
108}
109
110void
111ifq_start_task(void *p)
112{
113	struct ifqueue *ifq = p;
114	struct ifnet *ifp = ifq->ifq_if;
115
116	if (!ISSET(ifp->if_flags, IFF_RUNNING) ||
117	    ifq_empty(ifq) || ifq_is_oactive(ifq))
118		return;
119
120	ifp->if_qstart(ifq);
121}
122
123void
124ifq_restart_task(void *p)
125{
126	struct ifqueue *ifq = p;
127	struct ifnet *ifp = ifq->ifq_if;
128
129	ifq_clr_oactive(ifq);
130	ifp->if_qstart(ifq);
131}
132
133void
134ifq_barrier(struct ifqueue *ifq)
135{
136	struct cond c = COND_INITIALIZER();
137	struct task t = TASK_INITIALIZER(ifq_barrier_task, &c);
138
139	/* this should only be called from converted drivers */
140	KASSERT(ISSET(ifq->ifq_if->if_xflags, IFXF_MPSAFE));
141
142	if (ifq->ifq_serializer == NULL)
143		return;
144
145	ifq_serialize(ifq, &t);
146
147	cond_wait(&c, "ifqbar");
148}
149
150void
151ifq_barrier_task(void *p)
152{
153	unsigned int *notdone = p;
154
155	*notdone = 0;
156	wakeup_one(notdone);
157}
158
159/*
160 * ifqueue mbuf queue API
161 */
162
163void
164ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
165{
166	ifq->ifq_if = ifp;
167	ifq->ifq_softc = NULL;
168
169	mtx_init(&ifq->ifq_mtx, IPL_NET);
170	ifq->ifq_qdrops = 0;
171
172	/* default to priq */
173	ifq->ifq_ops = &priq_ops;
174	ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL);
175
176	ml_init(&ifq->ifq_free);
177	ifq->ifq_len = 0;
178
179	ifq->ifq_packets = 0;
180	ifq->ifq_bytes = 0;
181	ifq->ifq_qdrops = 0;
182	ifq->ifq_errors = 0;
183	ifq->ifq_mcasts = 0;
184
185	mtx_init(&ifq->ifq_task_mtx, IPL_NET);
186	TAILQ_INIT(&ifq->ifq_task_list);
187	ifq->ifq_serializer = NULL;
188
189	task_set(&ifq->ifq_start, ifq_start_task, ifq);
190	task_set(&ifq->ifq_restart, ifq_restart_task, ifq);
191
192	if (ifq->ifq_maxlen == 0)
193		ifq_set_maxlen(ifq, IFQ_MAXLEN);
194
195	ifq->ifq_idx = idx;
196}
197
198void
199ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg)
200{
201	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
202	struct mbuf_list free_ml = MBUF_LIST_INITIALIZER();
203	struct mbuf *m;
204	const struct ifq_ops *oldops;
205	void *newq, *oldq;
206
207	newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg);
208
209	mtx_enter(&ifq->ifq_mtx);
210	ifq->ifq_ops->ifqop_purge(ifq, &ml);
211	ifq->ifq_len = 0;
212
213	oldops = ifq->ifq_ops;
214	oldq = ifq->ifq_q;
215
216	ifq->ifq_ops = newops;
217	ifq->ifq_q = newq;
218
219	while ((m = ml_dequeue(&ml)) != NULL) {
220		m = ifq->ifq_ops->ifqop_enq(ifq, m);
221		if (m != NULL) {
222			ifq->ifq_qdrops++;
223			ml_enqueue(&free_ml, m);
224		} else
225			ifq->ifq_len++;
226	}
227	mtx_leave(&ifq->ifq_mtx);
228
229	oldops->ifqop_free(ifq->ifq_idx, oldq);
230
231	ml_purge(&free_ml);
232}
233
234void
235ifq_destroy(struct ifqueue *ifq)
236{
237	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
238
239	/* don't need to lock because this is the last use of the ifq */
240
241	ifq->ifq_ops->ifqop_purge(ifq, &ml);
242	ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q);
243
244	ml_purge(&ml);
245}
246
247void
248ifq_add_data(struct ifqueue *ifq, struct if_data *data)
249{
250	mtx_enter(&ifq->ifq_mtx);
251	data->ifi_opackets += ifq->ifq_packets;
252	data->ifi_obytes += ifq->ifq_bytes;
253	data->ifi_oqdrops += ifq->ifq_qdrops;
254	data->ifi_omcasts += ifq->ifq_mcasts;
255	/* ifp->if_data.ifi_oerrors */
256	mtx_leave(&ifq->ifq_mtx);
257}
258
259int
260ifq_enqueue(struct ifqueue *ifq, struct mbuf *m)
261{
262	struct mbuf *dm;
263
264	mtx_enter(&ifq->ifq_mtx);
265	dm = ifq->ifq_ops->ifqop_enq(ifq, m);
266	if (dm != m) {
267		ifq->ifq_packets++;
268		ifq->ifq_bytes += m->m_pkthdr.len;
269		if (ISSET(m->m_flags, M_MCAST))
270			ifq->ifq_mcasts++;
271	}
272
273	if (dm == NULL)
274		ifq->ifq_len++;
275	else
276		ifq->ifq_qdrops++;
277	mtx_leave(&ifq->ifq_mtx);
278
279	if (dm != NULL)
280		m_freem(dm);
281
282	return (dm == m ? ENOBUFS : 0);
283}
284
285static inline void
286ifq_deq_enter(struct ifqueue *ifq)
287{
288	mtx_enter(&ifq->ifq_mtx);
289}
290
291static inline void
292ifq_deq_leave(struct ifqueue *ifq)
293{
294	struct mbuf_list ml;
295
296	ml = ifq->ifq_free;
297	ml_init(&ifq->ifq_free);
298
299	mtx_leave(&ifq->ifq_mtx);
300
301	if (!ml_empty(&ml))
302		ml_purge(&ml);
303}
304
305struct mbuf *
306ifq_deq_begin(struct ifqueue *ifq)
307{
308	struct mbuf *m = NULL;
309	void *cookie;
310
311	ifq_deq_enter(ifq);
312	if (ifq->ifq_len == 0 ||
313	    (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) {
314		ifq_deq_leave(ifq);
315		return (NULL);
316	}
317
318	m->m_pkthdr.ph_cookie = cookie;
319
320	return (m);
321}
322
323void
324ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m)
325{
326	void *cookie;
327
328	KASSERT(m != NULL);
329	cookie = m->m_pkthdr.ph_cookie;
330
331	ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie);
332	ifq->ifq_len--;
333	ifq_deq_leave(ifq);
334}
335
336void
337ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m)
338{
339	KASSERT(m != NULL);
340
341	ifq_deq_leave(ifq);
342}
343
344struct mbuf *
345ifq_dequeue(struct ifqueue *ifq)
346{
347	struct mbuf *m;
348
349	m = ifq_deq_begin(ifq);
350	if (m == NULL)
351		return (NULL);
352
353	ifq_deq_commit(ifq, m);
354
355	return (m);
356}
357
358unsigned int
359ifq_purge(struct ifqueue *ifq)
360{
361	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
362	unsigned int rv;
363
364	mtx_enter(&ifq->ifq_mtx);
365	ifq->ifq_ops->ifqop_purge(ifq, &ml);
366	rv = ifq->ifq_len;
367	ifq->ifq_len = 0;
368	ifq->ifq_qdrops += rv;
369	mtx_leave(&ifq->ifq_mtx);
370
371	KASSERT(rv == ml_len(&ml));
372
373	ml_purge(&ml);
374
375	return (rv);
376}
377
378void *
379ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops)
380{
381	mtx_enter(&ifq->ifq_mtx);
382	if (ifq->ifq_ops == ops)
383		return (ifq->ifq_q);
384
385	mtx_leave(&ifq->ifq_mtx);
386
387	return (NULL);
388}
389
390void
391ifq_q_leave(struct ifqueue *ifq, void *q)
392{
393	KASSERT(q == ifq->ifq_q);
394	mtx_leave(&ifq->ifq_mtx);
395}
396
397void
398ifq_mfreem(struct ifqueue *ifq, struct mbuf *m)
399{
400	MUTEX_ASSERT_LOCKED(&ifq->ifq_mtx);
401
402	ifq->ifq_len--;
403	ifq->ifq_qdrops++;
404	ml_enqueue(&ifq->ifq_free, m);
405}
406
407void
408ifq_mfreeml(struct ifqueue *ifq, struct mbuf_list *ml)
409{
410	MUTEX_ASSERT_LOCKED(&ifq->ifq_mtx);
411
412	ifq->ifq_len -= ml_len(ml);
413	ifq->ifq_qdrops += ml_len(ml);
414	ml_enlist(&ifq->ifq_free, ml);
415}
416
417/*
418 * priq implementation
419 */
420
421unsigned int
422priq_idx(unsigned int nqueues, const struct mbuf *m)
423{
424	unsigned int flow = 0;
425
426	if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
427		flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
428
429	return (flow % nqueues);
430}
431
432void *
433priq_alloc(unsigned int idx, void *null)
434{
435	struct priq *pq;
436	int i;
437
438	pq = malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK);
439	for (i = 0; i < IFQ_NQUEUES; i++)
440		ml_init(&pq->pq_lists[i]);
441	return (pq);
442}
443
444void
445priq_free(unsigned int idx, void *pq)
446{
447	free(pq, M_DEVBUF, sizeof(struct priq));
448}
449
450struct mbuf *
451priq_enq(struct ifqueue *ifq, struct mbuf *m)
452{
453	struct priq *pq;
454	struct mbuf_list *pl;
455	struct mbuf *n = NULL;
456	unsigned int prio;
457
458	pq = ifq->ifq_q;
459	KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO);
460
461	/* Find a lower priority queue to drop from */
462	if (ifq_len(ifq) >= ifq->ifq_maxlen) {
463		for (prio = 0; prio < m->m_pkthdr.pf.prio; prio++) {
464			pl = &pq->pq_lists[prio];
465			if (ml_len(pl) > 0) {
466				n = ml_dequeue(pl);
467				goto enqueue;
468			}
469		}
470		/*
471		 * There's no lower priority queue that we can
472		 * drop from so don't enqueue this one.
473		 */
474		return (m);
475	}
476
477 enqueue:
478	pl = &pq->pq_lists[m->m_pkthdr.pf.prio];
479	ml_enqueue(pl, m);
480
481	return (n);
482}
483
484struct mbuf *
485priq_deq_begin(struct ifqueue *ifq, void **cookiep)
486{
487	struct priq *pq = ifq->ifq_q;
488	struct mbuf_list *pl;
489	unsigned int prio = nitems(pq->pq_lists);
490	struct mbuf *m;
491
492	do {
493		pl = &pq->pq_lists[--prio];
494		m = MBUF_LIST_FIRST(pl);
495		if (m != NULL) {
496			*cookiep = pl;
497			return (m);
498		}
499	} while (prio > 0);
500
501	return (NULL);
502}
503
504void
505priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie)
506{
507	struct mbuf_list *pl = cookie;
508
509	KASSERT(MBUF_LIST_FIRST(pl) == m);
510
511	ml_dequeue(pl);
512}
513
514void
515priq_purge(struct ifqueue *ifq, struct mbuf_list *ml)
516{
517	struct priq *pq = ifq->ifq_q;
518	struct mbuf_list *pl;
519	unsigned int prio = nitems(pq->pq_lists);
520
521	do {
522		pl = &pq->pq_lists[--prio];
523		ml_enlist(ml, pl);
524	} while (prio > 0);
525}
526