ifq.c revision 1.8 
1/*	$OpenBSD: ifq.c,v 1.8 2017/03/07 15:16:01 mikeb Exp $ */
2
3/*
4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/socket.h>
22#include <sys/mbuf.h>
23#include <sys/proc.h>
24
25#include <net/if.h>
26#include <net/if_var.h>
27
28/*
29 * priq glue
30 */
31unsigned int	 priq_idx(unsigned int, const struct mbuf *);
32struct mbuf	*priq_enq(struct ifqueue *, struct mbuf *);
33struct mbuf	*priq_deq_begin(struct ifqueue *, void **);
34void		 priq_deq_commit(struct ifqueue *, struct mbuf *, void *);
35void		 priq_purge(struct ifqueue *, struct mbuf_list *);
36
37void		*priq_alloc(unsigned int, void *);
38void		 priq_free(unsigned int, void *);
39
40const struct ifq_ops priq_ops = {
41	priq_idx,
42	priq_enq,
43	priq_deq_begin,
44	priq_deq_commit,
45	priq_purge,
46	priq_alloc,
47	priq_free,
48};
49
50const struct ifq_ops * const ifq_priq_ops = &priq_ops;
51
52/*
53 * priq internal structures
54 */
55
56struct priq {
57	struct mbuf_list	 pq_lists[IFQ_NQUEUES];
58};
59
60/*
61 * ifqueue serialiser
62 */
63
64void	ifq_start_task(void *);
65void	ifq_restart_task(void *);
66void	ifq_barrier_task(void *);
67
68#define TASK_ONQUEUE 0x1
69
70void
71ifq_serialize(struct ifqueue *ifq, struct task *t)
72{
73	struct task work;
74
75	if (ISSET(t->t_flags, TASK_ONQUEUE))
76		return;
77
78	mtx_enter(&ifq->ifq_task_mtx);
79	if (!ISSET(t->t_flags, TASK_ONQUEUE)) {
80		SET(t->t_flags, TASK_ONQUEUE);
81		TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry);
82	}
83
84	if (ifq->ifq_serializer == NULL) {
85		ifq->ifq_serializer = curcpu();
86
87		while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) {
88			TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry);
89			CLR(t->t_flags, TASK_ONQUEUE);
90			work = *t; /* copy to caller to avoid races */
91
92			mtx_leave(&ifq->ifq_task_mtx);
93
94			(*work.t_func)(work.t_arg);
95
96			mtx_enter(&ifq->ifq_task_mtx);
97		}
98
99		ifq->ifq_serializer = NULL;
100	}
101	mtx_leave(&ifq->ifq_task_mtx);
102}
103
104int
105ifq_is_serialized(struct ifqueue *ifq)
106{
107	return (ifq->ifq_serializer == curcpu());
108}
109
110void
111ifq_start_task(void *p)
112{
113	struct ifqueue *ifq = p;
114	struct ifnet *ifp = ifq->ifq_if;
115
116	if (!ISSET(ifp->if_flags, IFF_RUNNING) ||
117	    ifq_empty(ifq) || ifq_is_oactive(ifq))
118		return;
119
120	ifp->if_qstart(ifq);
121}
122
123void
124ifq_restart_task(void *p)
125{
126	struct ifqueue *ifq = p;
127	struct ifnet *ifp = ifq->ifq_if;
128
129	ifq_clr_oactive(ifq);
130	ifp->if_qstart(ifq);
131}
132
133void
134ifq_barrier(struct ifqueue *ifq)
135{
136	struct sleep_state sls;
137	unsigned int notdone = 1;
138	struct task t = TASK_INITIALIZER(ifq_barrier_task, &notdone);
139
140	/* this should only be called from converted drivers */
141	KASSERT(ISSET(ifq->ifq_if->if_xflags, IFXF_MPSAFE));
142
143	if (ifq->ifq_serializer == NULL)
144		return;
145
146	ifq_serialize(ifq, &t);
147
148	while (notdone) {
149		sleep_setup(&sls, &notdone, PWAIT, "ifqbar");
150		sleep_finish(&sls, notdone);
151	}
152}
153
154void
155ifq_barrier_task(void *p)
156{
157	unsigned int *notdone = p;
158
159	*notdone = 0;
160	wakeup_one(notdone);
161}
162
163/*
164 * ifqueue mbuf queue API
165 */
166
167void
168ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
169{
170	ifq->ifq_if = ifp;
171	ifq->ifq_softc = NULL;
172
173	mtx_init(&ifq->ifq_mtx, IPL_NET);
174	ifq->ifq_qdrops = 0;
175
176	/* default to priq */
177	ifq->ifq_ops = &priq_ops;
178	ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL);
179
180	ifq->ifq_len = 0;
181
182	ifq->ifq_packets = 0;
183	ifq->ifq_bytes = 0;
184	ifq->ifq_qdrops = 0;
185	ifq->ifq_errors = 0;
186	ifq->ifq_mcasts = 0;
187
188	mtx_init(&ifq->ifq_task_mtx, IPL_NET);
189	TAILQ_INIT(&ifq->ifq_task_list);
190	ifq->ifq_serializer = NULL;
191
192	task_set(&ifq->ifq_start, ifq_start_task, ifq);
193	task_set(&ifq->ifq_restart, ifq_restart_task, ifq);
194
195	if (ifq->ifq_maxlen == 0)
196		ifq_set_maxlen(ifq, IFQ_MAXLEN);
197
198	ifq->ifq_idx = idx;
199}
200
201void
202ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg)
203{
204	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
205	struct mbuf_list free_ml = MBUF_LIST_INITIALIZER();
206	struct mbuf *m;
207	const struct ifq_ops *oldops;
208	void *newq, *oldq;
209
210	newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg);
211
212	mtx_enter(&ifq->ifq_mtx);
213	ifq->ifq_ops->ifqop_purge(ifq, &ml);
214	ifq->ifq_len = 0;
215
216	oldops = ifq->ifq_ops;
217	oldq = ifq->ifq_q;
218
219	ifq->ifq_ops = newops;
220	ifq->ifq_q = newq;
221
222	while ((m = ml_dequeue(&ml)) != NULL) {
223		m = ifq->ifq_ops->ifqop_enq(ifq, m);
224		if (m != NULL) {
225			ifq->ifq_qdrops++;
226			ml_enqueue(&free_ml, m);
227		} else
228			ifq->ifq_len++;
229	}
230	mtx_leave(&ifq->ifq_mtx);
231
232	oldops->ifqop_free(ifq->ifq_idx, oldq);
233
234	ml_purge(&free_ml);
235}
236
237void
238ifq_destroy(struct ifqueue *ifq)
239{
240	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
241
242	/* don't need to lock because this is the last use of the ifq */
243
244	ifq->ifq_ops->ifqop_purge(ifq, &ml);
245	ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q);
246
247	ml_purge(&ml);
248}
249
250int
251ifq_enqueue(struct ifqueue *ifq, struct mbuf *m)
252{
253	struct mbuf *dm;
254
255	mtx_enter(&ifq->ifq_mtx);
256	dm = ifq->ifq_ops->ifqop_enq(ifq, m);
257	if (dm != m) {
258		ifq->ifq_packets++;
259		ifq->ifq_bytes += m->m_pkthdr.len;
260		if (ISSET(m->m_flags, M_MCAST))
261			ifq->ifq_mcasts++;
262	}
263
264	if (dm == NULL)
265		ifq->ifq_len++;
266	else
267		ifq->ifq_qdrops++;
268	mtx_leave(&ifq->ifq_mtx);
269
270	if (dm != NULL)
271		m_freem(dm);
272
273	return (dm == m ? ENOBUFS : 0);
274}
275
276struct mbuf *
277ifq_deq_begin(struct ifqueue *ifq)
278{
279	struct mbuf *m = NULL;
280	void *cookie;
281
282	mtx_enter(&ifq->ifq_mtx);
283	if (ifq->ifq_len == 0 ||
284	    (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) {
285		mtx_leave(&ifq->ifq_mtx);
286		return (NULL);
287	}
288
289	m->m_pkthdr.ph_cookie = cookie;
290
291	return (m);
292}
293
294void
295ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m)
296{
297	void *cookie;
298
299	KASSERT(m != NULL);
300	cookie = m->m_pkthdr.ph_cookie;
301
302	ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie);
303	ifq->ifq_len--;
304	mtx_leave(&ifq->ifq_mtx);
305}
306
307void
308ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m)
309{
310	KASSERT(m != NULL);
311
312	mtx_leave(&ifq->ifq_mtx);
313}
314
315struct mbuf *
316ifq_dequeue(struct ifqueue *ifq)
317{
318	struct mbuf *m;
319
320	m = ifq_deq_begin(ifq);
321	if (m == NULL)
322		return (NULL);
323
324	ifq_deq_commit(ifq, m);
325
326	return (m);
327}
328
329unsigned int
330ifq_purge(struct ifqueue *ifq)
331{
332	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
333	unsigned int rv;
334
335	mtx_enter(&ifq->ifq_mtx);
336	ifq->ifq_ops->ifqop_purge(ifq, &ml);
337	rv = ifq->ifq_len;
338	ifq->ifq_len = 0;
339	ifq->ifq_qdrops += rv;
340	mtx_leave(&ifq->ifq_mtx);
341
342	KASSERT(rv == ml_len(&ml));
343
344	ml_purge(&ml);
345
346	return (rv);
347}
348
349void *
350ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops)
351{
352	mtx_enter(&ifq->ifq_mtx);
353	if (ifq->ifq_ops == ops)
354		return (ifq->ifq_q);
355
356	mtx_leave(&ifq->ifq_mtx);
357
358	return (NULL);
359}
360
361void
362ifq_q_leave(struct ifqueue *ifq, void *q)
363{
364	KASSERT(q == ifq->ifq_q);
365	mtx_leave(&ifq->ifq_mtx);
366}
367
368/*
369 * priq implementation
370 */
371
372unsigned int
373priq_idx(unsigned int nqueues, const struct mbuf *m)
374{
375	unsigned int flow = 0;
376
377	if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
378		flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
379
380	return (flow % nqueues);
381}
382
383void *
384priq_alloc(unsigned int idx, void *null)
385{
386	struct priq *pq;
387	int i;
388
389	pq = malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK);
390	for (i = 0; i < IFQ_NQUEUES; i++)
391		ml_init(&pq->pq_lists[i]);
392	return (pq);
393}
394
395void
396priq_free(unsigned int idx, void *pq)
397{
398	free(pq, M_DEVBUF, sizeof(struct priq));
399}
400
401struct mbuf *
402priq_enq(struct ifqueue *ifq, struct mbuf *m)
403{
404	struct priq *pq;
405	struct mbuf_list *pl;
406
407	if (ifq_len(ifq) >= ifq->ifq_maxlen)
408		return (m);
409
410	pq = ifq->ifq_q;
411	KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO);
412	pl = &pq->pq_lists[m->m_pkthdr.pf.prio];
413
414	ml_enqueue(pl, m);
415
416	return (NULL);
417}
418
419struct mbuf *
420priq_deq_begin(struct ifqueue *ifq, void **cookiep)
421{
422	struct priq *pq = ifq->ifq_q;
423	struct mbuf_list *pl;
424	unsigned int prio = nitems(pq->pq_lists);
425	struct mbuf *m;
426
427	do {
428		pl = &pq->pq_lists[--prio];
429		m = MBUF_LIST_FIRST(pl);
430		if (m != NULL) {
431			*cookiep = pl;
432			return (m);
433		}
434	} while (prio > 0);
435
436	return (NULL);
437}
438
439void
440priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie)
441{
442	struct mbuf_list *pl = cookie;
443
444	KASSERT(MBUF_LIST_FIRST(pl) == m);
445
446	ml_dequeue(pl);
447}
448
449void
450priq_purge(struct ifqueue *ifq, struct mbuf_list *ml)
451{
452	struct priq *pq = ifq->ifq_q;
453	struct mbuf_list *pl;
454	unsigned int prio = nitems(pq->pq_lists);
455
456	do {
457		pl = &pq->pq_lists[--prio];
458		ml_enlist(ml, pl);
459	} while (prio > 0);
460}
461