ifq.c revision 1.6 
1/*	$OpenBSD: ifq.c,v 1.6 2017/01/24 03:57:35 dlg Exp $ */
2
3/*
4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/socket.h>
22#include <sys/mbuf.h>
23#include <sys/proc.h>
24
25#include <net/if.h>
26#include <net/if_var.h>
27
28/*
29 * priq glue
30 */
31unsigned int	 priq_idx(unsigned int, const struct mbuf *);
32int		 priq_enq(struct ifqueue *, struct mbuf *);
33struct mbuf	*priq_deq_begin(struct ifqueue *, void **);
34void		 priq_deq_commit(struct ifqueue *, struct mbuf *, void *);
35void		 priq_purge(struct ifqueue *, struct mbuf_list *);
36
37void		*priq_alloc(unsigned int, void *);
38void		 priq_free(unsigned int, void *);
39
40const struct ifq_ops priq_ops = {
41	priq_idx,
42	priq_enq,
43	priq_deq_begin,
44	priq_deq_commit,
45	priq_purge,
46	priq_alloc,
47	priq_free,
48};
49
50const struct ifq_ops * const ifq_priq_ops = &priq_ops;
51
52/*
53 * priq internal structures
54 */
55
56struct priq_list {
57	struct mbuf		*head;
58	struct mbuf		*tail;
59};
60
61struct priq {
62	struct priq_list	 pq_lists[IFQ_NQUEUES];
63};
64
65/*
66 * ifqueue serialiser
67 */
68
69void	ifq_start_task(void *);
70void	ifq_restart_task(void *);
71void	ifq_barrier_task(void *);
72
73#define TASK_ONQUEUE 0x1
74
75void
76ifq_serialize(struct ifqueue *ifq, struct task *t)
77{
78	struct task work;
79
80	if (ISSET(t->t_flags, TASK_ONQUEUE))
81		return;
82
83	mtx_enter(&ifq->ifq_task_mtx);
84	if (!ISSET(t->t_flags, TASK_ONQUEUE)) {
85		SET(t->t_flags, TASK_ONQUEUE);
86		TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry);
87	}
88
89	if (ifq->ifq_serializer == NULL) {
90		ifq->ifq_serializer = curcpu();
91
92		while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) {
93			TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry);
94			CLR(t->t_flags, TASK_ONQUEUE);
95			work = *t; /* copy to caller to avoid races */
96
97			mtx_leave(&ifq->ifq_task_mtx);
98
99			(*work.t_func)(work.t_arg);
100
101			mtx_enter(&ifq->ifq_task_mtx);
102		}
103
104		ifq->ifq_serializer = NULL;
105	}
106	mtx_leave(&ifq->ifq_task_mtx);
107}
108
109int
110ifq_is_serialized(struct ifqueue *ifq)
111{
112	return (ifq->ifq_serializer == curcpu());
113}
114
115void
116ifq_start_task(void *p)
117{
118	struct ifqueue *ifq = p;
119	struct ifnet *ifp = ifq->ifq_if;
120
121	if (!ISSET(ifp->if_flags, IFF_RUNNING) ||
122	    ifq_empty(ifq) || ifq_is_oactive(ifq))
123		return;
124
125	ifp->if_qstart(ifq);
126}
127
128void
129ifq_restart_task(void *p)
130{
131	struct ifqueue *ifq = p;
132	struct ifnet *ifp = ifq->ifq_if;
133
134	ifq_clr_oactive(ifq);
135	ifp->if_qstart(ifq);
136}
137
138void
139ifq_barrier(struct ifqueue *ifq)
140{
141	struct sleep_state sls;
142	unsigned int notdone = 1;
143	struct task t = TASK_INITIALIZER(ifq_barrier_task, &notdone);
144
145	/* this should only be called from converted drivers */
146	KASSERT(ISSET(ifq->ifq_if->if_xflags, IFXF_MPSAFE));
147
148	if (ifq->ifq_serializer == NULL)
149		return;
150
151	ifq_serialize(ifq, &t);
152
153	while (notdone) {
154		sleep_setup(&sls, &notdone, PWAIT, "ifqbar");
155		sleep_finish(&sls, notdone);
156	}
157}
158
159void
160ifq_barrier_task(void *p)
161{
162	unsigned int *notdone = p;
163
164	*notdone = 0;
165	wakeup_one(notdone);
166}
167
168/*
169 * ifqueue mbuf queue API
170 */
171
172void
173ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
174{
175	ifq->ifq_if = ifp;
176	ifq->ifq_softc = NULL;
177
178	mtx_init(&ifq->ifq_mtx, IPL_NET);
179	ifq->ifq_qdrops = 0;
180
181	/* default to priq */
182	ifq->ifq_ops = &priq_ops;
183	ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL);
184
185	ifq->ifq_len = 0;
186
187	ifq->ifq_packets = 0;
188	ifq->ifq_bytes = 0;
189	ifq->ifq_qdrops = 0;
190	ifq->ifq_errors = 0;
191	ifq->ifq_mcasts = 0;
192
193	mtx_init(&ifq->ifq_task_mtx, IPL_NET);
194	TAILQ_INIT(&ifq->ifq_task_list);
195	ifq->ifq_serializer = NULL;
196
197	task_set(&ifq->ifq_start, ifq_start_task, ifq);
198	task_set(&ifq->ifq_restart, ifq_restart_task, ifq);
199
200	if (ifq->ifq_maxlen == 0)
201		ifq_set_maxlen(ifq, IFQ_MAXLEN);
202
203	ifq->ifq_idx = idx;
204}
205
206void
207ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg)
208{
209	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
210	struct mbuf_list free_ml = MBUF_LIST_INITIALIZER();
211	struct mbuf *m;
212	const struct ifq_ops *oldops;
213	void *newq, *oldq;
214
215	newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg);
216
217	mtx_enter(&ifq->ifq_mtx);
218	ifq->ifq_ops->ifqop_purge(ifq, &ml);
219	ifq->ifq_len = 0;
220
221	oldops = ifq->ifq_ops;
222	oldq = ifq->ifq_q;
223
224	ifq->ifq_ops = newops;
225	ifq->ifq_q = newq;
226
227	while ((m = ml_dequeue(&ml)) != NULL) {
228		if (ifq->ifq_ops->ifqop_enq(ifq, m) != 0) {
229			ifq->ifq_qdrops++;
230			ml_enqueue(&free_ml, m);
231		} else
232			ifq->ifq_len++;
233	}
234	mtx_leave(&ifq->ifq_mtx);
235
236	oldops->ifqop_free(ifq->ifq_idx, oldq);
237
238	ml_purge(&free_ml);
239}
240
241void
242ifq_destroy(struct ifqueue *ifq)
243{
244	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
245
246	/* don't need to lock because this is the last use of the ifq */
247
248	ifq->ifq_ops->ifqop_purge(ifq, &ml);
249	ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q);
250
251	ml_purge(&ml);
252}
253
254int
255ifq_enqueue_try(struct ifqueue *ifq, struct mbuf *m)
256{
257	int rv;
258
259	mtx_enter(&ifq->ifq_mtx);
260	rv = ifq->ifq_ops->ifqop_enq(ifq, m);
261	if (rv == 0) {
262		ifq->ifq_len++;
263
264		ifq->ifq_packets++;
265		ifq->ifq_bytes += m->m_pkthdr.len;
266		if (ISSET(m->m_flags, M_MCAST))
267			ifq->ifq_mcasts++;
268	} else
269		ifq->ifq_qdrops++;
270	mtx_leave(&ifq->ifq_mtx);
271
272	return (rv);
273}
274
275int
276ifq_enqueue(struct ifqueue *ifq, struct mbuf *m)
277{
278	int err;
279
280	err = ifq_enqueue_try(ifq, m);
281	if (err != 0)
282		m_freem(m);
283
284	return (err);
285}
286
287struct mbuf *
288ifq_deq_begin(struct ifqueue *ifq)
289{
290	struct mbuf *m = NULL;
291	void *cookie;
292
293	mtx_enter(&ifq->ifq_mtx);
294	if (ifq->ifq_len == 0 ||
295	    (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) {
296		mtx_leave(&ifq->ifq_mtx);
297		return (NULL);
298	}
299
300	m->m_pkthdr.ph_cookie = cookie;
301
302	return (m);
303}
304
305void
306ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m)
307{
308	void *cookie;
309
310	KASSERT(m != NULL);
311	cookie = m->m_pkthdr.ph_cookie;
312
313	ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie);
314	ifq->ifq_len--;
315	mtx_leave(&ifq->ifq_mtx);
316}
317
318void
319ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m)
320{
321	KASSERT(m != NULL);
322
323	mtx_leave(&ifq->ifq_mtx);
324}
325
326struct mbuf *
327ifq_dequeue(struct ifqueue *ifq)
328{
329	struct mbuf *m;
330
331	m = ifq_deq_begin(ifq);
332	if (m == NULL)
333		return (NULL);
334
335	ifq_deq_commit(ifq, m);
336
337	return (m);
338}
339
340unsigned int
341ifq_purge(struct ifqueue *ifq)
342{
343	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
344	unsigned int rv;
345
346	mtx_enter(&ifq->ifq_mtx);
347	ifq->ifq_ops->ifqop_purge(ifq, &ml);
348	rv = ifq->ifq_len;
349	ifq->ifq_len = 0;
350	ifq->ifq_qdrops += rv;
351	mtx_leave(&ifq->ifq_mtx);
352
353	KASSERT(rv == ml_len(&ml));
354
355	ml_purge(&ml);
356
357	return (rv);
358}
359
360void *
361ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops)
362{
363	mtx_enter(&ifq->ifq_mtx);
364	if (ifq->ifq_ops == ops)
365		return (ifq->ifq_q);
366
367	mtx_leave(&ifq->ifq_mtx);
368
369	return (NULL);
370}
371
372void
373ifq_q_leave(struct ifqueue *ifq, void *q)
374{
375	KASSERT(q == ifq->ifq_q);
376	mtx_leave(&ifq->ifq_mtx);
377}
378
379/*
380 * priq implementation
381 */
382
383unsigned int
384priq_idx(unsigned int nqueues, const struct mbuf *m)
385{
386	unsigned int flow = 0;
387
388	if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
389		flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
390
391	return (flow % nqueues);
392}
393
394void *
395priq_alloc(unsigned int idx, void *null)
396{
397	return (malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK | M_ZERO));
398}
399
400void
401priq_free(unsigned int idx, void *pq)
402{
403	free(pq, M_DEVBUF, sizeof(struct priq));
404}
405
406int
407priq_enq(struct ifqueue *ifq, struct mbuf *m)
408{
409	struct priq *pq;
410	struct priq_list *pl;
411
412	if (ifq_len(ifq) >= ifq->ifq_maxlen)
413		return (ENOBUFS);
414
415	pq = ifq->ifq_q;
416	KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO);
417	pl = &pq->pq_lists[m->m_pkthdr.pf.prio];
418
419	m->m_nextpkt = NULL;
420	if (pl->tail == NULL)
421		pl->head = m;
422	else
423		pl->tail->m_nextpkt = m;
424	pl->tail = m;
425
426	return (0);
427}
428
429struct mbuf *
430priq_deq_begin(struct ifqueue *ifq, void **cookiep)
431{
432	struct priq *pq = ifq->ifq_q;
433	struct priq_list *pl;
434	unsigned int prio = nitems(pq->pq_lists);
435	struct mbuf *m;
436
437	do {
438		pl = &pq->pq_lists[--prio];
439		m = pl->head;
440		if (m != NULL) {
441			*cookiep = pl;
442			return (m);
443		}
444	} while (prio > 0);
445
446	return (NULL);
447}
448
449void
450priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie)
451{
452	struct priq_list *pl = cookie;
453
454	KASSERT(pl->head == m);
455
456	pl->head = m->m_nextpkt;
457	m->m_nextpkt = NULL;
458
459	if (pl->head == NULL)
460		pl->tail = NULL;
461}
462
463void
464priq_purge(struct ifqueue *ifq, struct mbuf_list *ml)
465{
466	struct priq *pq = ifq->ifq_q;
467	struct priq_list *pl;
468	unsigned int prio = nitems(pq->pq_lists);
469	struct mbuf *m, *n;
470
471	do {
472		pl = &pq->pq_lists[--prio];
473
474		for (m = pl->head; m != NULL; m = n) {
475			n = m->m_nextpkt;
476			ml_enqueue(ml, m);
477		}
478
479		pl->head = pl->tail = NULL;
480	} while (prio > 0);
481}
482