ifq.c revision 1.7 
1/*	$OpenBSD: ifq.c,v 1.7 2017/03/07 01:29:53 dlg Exp $ */
2
3/*
4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/socket.h>
22#include <sys/mbuf.h>
23#include <sys/proc.h>
24
25#include <net/if.h>
26#include <net/if_var.h>
27
28/*
29 * priq glue
30 */
31unsigned int	 priq_idx(unsigned int, const struct mbuf *);
32struct mbuf	*priq_enq(struct ifqueue *, struct mbuf *);
33struct mbuf	*priq_deq_begin(struct ifqueue *, void **);
34void		 priq_deq_commit(struct ifqueue *, struct mbuf *, void *);
35void		 priq_purge(struct ifqueue *, struct mbuf_list *);
36
37void		*priq_alloc(unsigned int, void *);
38void		 priq_free(unsigned int, void *);
39
40const struct ifq_ops priq_ops = {
41	priq_idx,
42	priq_enq,
43	priq_deq_begin,
44	priq_deq_commit,
45	priq_purge,
46	priq_alloc,
47	priq_free,
48};
49
50const struct ifq_ops * const ifq_priq_ops = &priq_ops;
51
52/*
53 * priq internal structures
54 */
55
56struct priq_list {
57	struct mbuf		*head;
58	struct mbuf		*tail;
59};
60
61struct priq {
62	struct priq_list	 pq_lists[IFQ_NQUEUES];
63};
64
65/*
66 * ifqueue serialiser
67 */
68
69void	ifq_start_task(void *);
70void	ifq_restart_task(void *);
71void	ifq_barrier_task(void *);
72
73#define TASK_ONQUEUE 0x1
74
75void
76ifq_serialize(struct ifqueue *ifq, struct task *t)
77{
78	struct task work;
79
80	if (ISSET(t->t_flags, TASK_ONQUEUE))
81		return;
82
83	mtx_enter(&ifq->ifq_task_mtx);
84	if (!ISSET(t->t_flags, TASK_ONQUEUE)) {
85		SET(t->t_flags, TASK_ONQUEUE);
86		TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry);
87	}
88
89	if (ifq->ifq_serializer == NULL) {
90		ifq->ifq_serializer = curcpu();
91
92		while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) {
93			TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry);
94			CLR(t->t_flags, TASK_ONQUEUE);
95			work = *t; /* copy to caller to avoid races */
96
97			mtx_leave(&ifq->ifq_task_mtx);
98
99			(*work.t_func)(work.t_arg);
100
101			mtx_enter(&ifq->ifq_task_mtx);
102		}
103
104		ifq->ifq_serializer = NULL;
105	}
106	mtx_leave(&ifq->ifq_task_mtx);
107}
108
109int
110ifq_is_serialized(struct ifqueue *ifq)
111{
112	return (ifq->ifq_serializer == curcpu());
113}
114
115void
116ifq_start_task(void *p)
117{
118	struct ifqueue *ifq = p;
119	struct ifnet *ifp = ifq->ifq_if;
120
121	if (!ISSET(ifp->if_flags, IFF_RUNNING) ||
122	    ifq_empty(ifq) || ifq_is_oactive(ifq))
123		return;
124
125	ifp->if_qstart(ifq);
126}
127
128void
129ifq_restart_task(void *p)
130{
131	struct ifqueue *ifq = p;
132	struct ifnet *ifp = ifq->ifq_if;
133
134	ifq_clr_oactive(ifq);
135	ifp->if_qstart(ifq);
136}
137
138void
139ifq_barrier(struct ifqueue *ifq)
140{
141	struct sleep_state sls;
142	unsigned int notdone = 1;
143	struct task t = TASK_INITIALIZER(ifq_barrier_task, &notdone);
144
145	/* this should only be called from converted drivers */
146	KASSERT(ISSET(ifq->ifq_if->if_xflags, IFXF_MPSAFE));
147
148	if (ifq->ifq_serializer == NULL)
149		return;
150
151	ifq_serialize(ifq, &t);
152
153	while (notdone) {
154		sleep_setup(&sls, &notdone, PWAIT, "ifqbar");
155		sleep_finish(&sls, notdone);
156	}
157}
158
159void
160ifq_barrier_task(void *p)
161{
162	unsigned int *notdone = p;
163
164	*notdone = 0;
165	wakeup_one(notdone);
166}
167
168/*
169 * ifqueue mbuf queue API
170 */
171
172void
173ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
174{
175	ifq->ifq_if = ifp;
176	ifq->ifq_softc = NULL;
177
178	mtx_init(&ifq->ifq_mtx, IPL_NET);
179	ifq->ifq_qdrops = 0;
180
181	/* default to priq */
182	ifq->ifq_ops = &priq_ops;
183	ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL);
184
185	ifq->ifq_len = 0;
186
187	ifq->ifq_packets = 0;
188	ifq->ifq_bytes = 0;
189	ifq->ifq_qdrops = 0;
190	ifq->ifq_errors = 0;
191	ifq->ifq_mcasts = 0;
192
193	mtx_init(&ifq->ifq_task_mtx, IPL_NET);
194	TAILQ_INIT(&ifq->ifq_task_list);
195	ifq->ifq_serializer = NULL;
196
197	task_set(&ifq->ifq_start, ifq_start_task, ifq);
198	task_set(&ifq->ifq_restart, ifq_restart_task, ifq);
199
200	if (ifq->ifq_maxlen == 0)
201		ifq_set_maxlen(ifq, IFQ_MAXLEN);
202
203	ifq->ifq_idx = idx;
204}
205
206void
207ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg)
208{
209	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
210	struct mbuf_list free_ml = MBUF_LIST_INITIALIZER();
211	struct mbuf *m;
212	const struct ifq_ops *oldops;
213	void *newq, *oldq;
214
215	newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg);
216
217	mtx_enter(&ifq->ifq_mtx);
218	ifq->ifq_ops->ifqop_purge(ifq, &ml);
219	ifq->ifq_len = 0;
220
221	oldops = ifq->ifq_ops;
222	oldq = ifq->ifq_q;
223
224	ifq->ifq_ops = newops;
225	ifq->ifq_q = newq;
226
227	while ((m = ml_dequeue(&ml)) != NULL) {
228		m = ifq->ifq_ops->ifqop_enq(ifq, m);
229		if (m != NULL) {
230			ifq->ifq_qdrops++;
231			ml_enqueue(&free_ml, m);
232		} else
233			ifq->ifq_len++;
234	}
235	mtx_leave(&ifq->ifq_mtx);
236
237	oldops->ifqop_free(ifq->ifq_idx, oldq);
238
239	ml_purge(&free_ml);
240}
241
242void
243ifq_destroy(struct ifqueue *ifq)
244{
245	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
246
247	/* don't need to lock because this is the last use of the ifq */
248
249	ifq->ifq_ops->ifqop_purge(ifq, &ml);
250	ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q);
251
252	ml_purge(&ml);
253}
254
255int
256ifq_enqueue(struct ifqueue *ifq, struct mbuf *m)
257{
258	struct mbuf *dm;
259
260	mtx_enter(&ifq->ifq_mtx);
261	dm = ifq->ifq_ops->ifqop_enq(ifq, m);
262	if (dm != m) {
263		ifq->ifq_packets++;
264		ifq->ifq_bytes += m->m_pkthdr.len;
265		if (ISSET(m->m_flags, M_MCAST))
266			ifq->ifq_mcasts++;
267	}
268
269	if (dm == NULL)
270		ifq->ifq_len++;
271	else
272		ifq->ifq_qdrops++;
273	mtx_leave(&ifq->ifq_mtx);
274
275	if (dm != NULL)
276		m_freem(dm);
277
278	return (dm == m ? ENOBUFS : 0);
279}
280
281struct mbuf *
282ifq_deq_begin(struct ifqueue *ifq)
283{
284	struct mbuf *m = NULL;
285	void *cookie;
286
287	mtx_enter(&ifq->ifq_mtx);
288	if (ifq->ifq_len == 0 ||
289	    (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) {
290		mtx_leave(&ifq->ifq_mtx);
291		return (NULL);
292	}
293
294	m->m_pkthdr.ph_cookie = cookie;
295
296	return (m);
297}
298
299void
300ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m)
301{
302	void *cookie;
303
304	KASSERT(m != NULL);
305	cookie = m->m_pkthdr.ph_cookie;
306
307	ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie);
308	ifq->ifq_len--;
309	mtx_leave(&ifq->ifq_mtx);
310}
311
312void
313ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m)
314{
315	KASSERT(m != NULL);
316
317	mtx_leave(&ifq->ifq_mtx);
318}
319
320struct mbuf *
321ifq_dequeue(struct ifqueue *ifq)
322{
323	struct mbuf *m;
324
325	m = ifq_deq_begin(ifq);
326	if (m == NULL)
327		return (NULL);
328
329	ifq_deq_commit(ifq, m);
330
331	return (m);
332}
333
334unsigned int
335ifq_purge(struct ifqueue *ifq)
336{
337	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
338	unsigned int rv;
339
340	mtx_enter(&ifq->ifq_mtx);
341	ifq->ifq_ops->ifqop_purge(ifq, &ml);
342	rv = ifq->ifq_len;
343	ifq->ifq_len = 0;
344	ifq->ifq_qdrops += rv;
345	mtx_leave(&ifq->ifq_mtx);
346
347	KASSERT(rv == ml_len(&ml));
348
349	ml_purge(&ml);
350
351	return (rv);
352}
353
354void *
355ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops)
356{
357	mtx_enter(&ifq->ifq_mtx);
358	if (ifq->ifq_ops == ops)
359		return (ifq->ifq_q);
360
361	mtx_leave(&ifq->ifq_mtx);
362
363	return (NULL);
364}
365
366void
367ifq_q_leave(struct ifqueue *ifq, void *q)
368{
369	KASSERT(q == ifq->ifq_q);
370	mtx_leave(&ifq->ifq_mtx);
371}
372
373/*
374 * priq implementation
375 */
376
377unsigned int
378priq_idx(unsigned int nqueues, const struct mbuf *m)
379{
380	unsigned int flow = 0;
381
382	if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
383		flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
384
385	return (flow % nqueues);
386}
387
388void *
389priq_alloc(unsigned int idx, void *null)
390{
391	return (malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK | M_ZERO));
392}
393
394void
395priq_free(unsigned int idx, void *pq)
396{
397	free(pq, M_DEVBUF, sizeof(struct priq));
398}
399
400struct mbuf *
401priq_enq(struct ifqueue *ifq, struct mbuf *m)
402{
403	struct priq *pq;
404	struct priq_list *pl;
405
406	if (ifq_len(ifq) >= ifq->ifq_maxlen)
407		return (m);
408
409	pq = ifq->ifq_q;
410	KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO);
411	pl = &pq->pq_lists[m->m_pkthdr.pf.prio];
412
413	m->m_nextpkt = NULL;
414	if (pl->tail == NULL)
415		pl->head = m;
416	else
417		pl->tail->m_nextpkt = m;
418	pl->tail = m;
419
420	return (NULL);
421}
422
423struct mbuf *
424priq_deq_begin(struct ifqueue *ifq, void **cookiep)
425{
426	struct priq *pq = ifq->ifq_q;
427	struct priq_list *pl;
428	unsigned int prio = nitems(pq->pq_lists);
429	struct mbuf *m;
430
431	do {
432		pl = &pq->pq_lists[--prio];
433		m = pl->head;
434		if (m != NULL) {
435			*cookiep = pl;
436			return (m);
437		}
438	} while (prio > 0);
439
440	return (NULL);
441}
442
443void
444priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie)
445{
446	struct priq_list *pl = cookie;
447
448	KASSERT(pl->head == m);
449
450	pl->head = m->m_nextpkt;
451	m->m_nextpkt = NULL;
452
453	if (pl->head == NULL)
454		pl->tail = NULL;
455}
456
457void
458priq_purge(struct ifqueue *ifq, struct mbuf_list *ml)
459{
460	struct priq *pq = ifq->ifq_q;
461	struct priq_list *pl;
462	unsigned int prio = nitems(pq->pq_lists);
463	struct mbuf *m, *n;
464
465	do {
466		pl = &pq->pq_lists[--prio];
467
468		for (m = pl->head; m != NULL; m = n) {
469			n = m->m_nextpkt;
470			ml_enqueue(ml, m);
471		}
472
473		pl->head = pl->tail = NULL;
474	} while (prio > 0);
475}
476