ifq.c revision 1.15 
1/*	$OpenBSD: ifq.c,v 1.15 2017/11/14 08:44:11 dlg Exp $ */
2
3/*
4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/socket.h>
22#include <sys/mbuf.h>
23#include <sys/proc.h>
24
25#include <net/if.h>
26#include <net/if_var.h>
27
28/*
29 * priq glue
30 */
31unsigned int	 priq_idx(unsigned int, const struct mbuf *);
32struct mbuf	*priq_enq(struct ifqueue *, struct mbuf *);
33struct mbuf	*priq_deq_begin(struct ifqueue *, void **);
34void		 priq_deq_commit(struct ifqueue *, struct mbuf *, void *);
35void		 priq_purge(struct ifqueue *, struct mbuf_list *);
36
37void		*priq_alloc(unsigned int, void *);
38void		 priq_free(unsigned int, void *);
39
40const struct ifq_ops priq_ops = {
41	priq_idx,
42	priq_enq,
43	priq_deq_begin,
44	priq_deq_commit,
45	priq_purge,
46	priq_alloc,
47	priq_free,
48};
49
50const struct ifq_ops * const ifq_priq_ops = &priq_ops;
51
52/*
53 * priq internal structures
54 */
55
56struct priq {
57	struct mbuf_list	 pq_lists[IFQ_NQUEUES];
58};
59
60/*
61 * ifqueue serialiser
62 */
63
64void	ifq_start_task(void *);
65void	ifq_restart_task(void *);
66void	ifq_barrier_task(void *);
67
68#define TASK_ONQUEUE 0x1
69
70void
71ifq_serialize(struct ifqueue *ifq, struct task *t)
72{
73	struct task work;
74
75	if (ISSET(t->t_flags, TASK_ONQUEUE))
76		return;
77
78	mtx_enter(&ifq->ifq_task_mtx);
79	if (!ISSET(t->t_flags, TASK_ONQUEUE)) {
80		SET(t->t_flags, TASK_ONQUEUE);
81		TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry);
82	}
83
84	if (ifq->ifq_serializer == NULL) {
85		ifq->ifq_serializer = curcpu();
86
87		while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) {
88			TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry);
89			CLR(t->t_flags, TASK_ONQUEUE);
90			work = *t; /* copy to caller to avoid races */
91
92			mtx_leave(&ifq->ifq_task_mtx);
93
94			(*work.t_func)(work.t_arg);
95
96			mtx_enter(&ifq->ifq_task_mtx);
97		}
98
99		ifq->ifq_serializer = NULL;
100	}
101	mtx_leave(&ifq->ifq_task_mtx);
102}
103
104int
105ifq_is_serialized(struct ifqueue *ifq)
106{
107	return (ifq->ifq_serializer == curcpu());
108}
109
110void
111ifq_start_task(void *p)
112{
113	struct ifqueue *ifq = p;
114	struct ifnet *ifp = ifq->ifq_if;
115
116	if (!ISSET(ifp->if_flags, IFF_RUNNING) ||
117	    ifq_empty(ifq) || ifq_is_oactive(ifq))
118		return;
119
120	ifp->if_qstart(ifq);
121}
122
123void
124ifq_restart_task(void *p)
125{
126	struct ifqueue *ifq = p;
127	struct ifnet *ifp = ifq->ifq_if;
128
129	ifq_clr_oactive(ifq);
130	ifp->if_qstart(ifq);
131}
132
133void
134ifq_barrier(struct ifqueue *ifq)
135{
136	struct sleep_state sls;
137	unsigned int notdone = 1;
138	struct task t = TASK_INITIALIZER(ifq_barrier_task, &notdone);
139
140	/* this should only be called from converted drivers */
141	KASSERT(ISSET(ifq->ifq_if->if_xflags, IFXF_MPSAFE));
142
143	if (ifq->ifq_serializer == NULL)
144		return;
145
146	ifq_serialize(ifq, &t);
147
148	while (notdone) {
149		sleep_setup(&sls, &notdone, PWAIT, "ifqbar");
150		sleep_finish(&sls, notdone);
151	}
152}
153
154void
155ifq_barrier_task(void *p)
156{
157	unsigned int *notdone = p;
158
159	*notdone = 0;
160	wakeup_one(notdone);
161}
162
163/*
164 * ifqueue mbuf queue API
165 */
166
167void
168ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
169{
170	ifq->ifq_if = ifp;
171	ifq->ifq_softc = NULL;
172
173	mtx_init(&ifq->ifq_mtx, IPL_NET);
174	ifq->ifq_qdrops = 0;
175
176	/* default to priq */
177	ifq->ifq_ops = &priq_ops;
178	ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL);
179
180	ml_init(&ifq->ifq_free);
181	ifq->ifq_len = 0;
182
183	ifq->ifq_packets = 0;
184	ifq->ifq_bytes = 0;
185	ifq->ifq_qdrops = 0;
186	ifq->ifq_errors = 0;
187	ifq->ifq_mcasts = 0;
188
189	mtx_init(&ifq->ifq_task_mtx, IPL_NET);
190	TAILQ_INIT(&ifq->ifq_task_list);
191	ifq->ifq_serializer = NULL;
192
193	task_set(&ifq->ifq_start, ifq_start_task, ifq);
194	task_set(&ifq->ifq_restart, ifq_restart_task, ifq);
195
196	if (ifq->ifq_maxlen == 0)
197		ifq_set_maxlen(ifq, IFQ_MAXLEN);
198
199	ifq->ifq_idx = idx;
200}
201
202void
203ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg)
204{
205	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
206	struct mbuf_list free_ml = MBUF_LIST_INITIALIZER();
207	struct mbuf *m;
208	const struct ifq_ops *oldops;
209	void *newq, *oldq;
210
211	newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg);
212
213	mtx_enter(&ifq->ifq_mtx);
214	ifq->ifq_ops->ifqop_purge(ifq, &ml);
215	ifq->ifq_len = 0;
216
217	oldops = ifq->ifq_ops;
218	oldq = ifq->ifq_q;
219
220	ifq->ifq_ops = newops;
221	ifq->ifq_q = newq;
222
223	while ((m = ml_dequeue(&ml)) != NULL) {
224		m = ifq->ifq_ops->ifqop_enq(ifq, m);
225		if (m != NULL) {
226			ifq->ifq_qdrops++;
227			ml_enqueue(&free_ml, m);
228		} else
229			ifq->ifq_len++;
230	}
231	mtx_leave(&ifq->ifq_mtx);
232
233	oldops->ifqop_free(ifq->ifq_idx, oldq);
234
235	ml_purge(&free_ml);
236}
237
238void
239ifq_destroy(struct ifqueue *ifq)
240{
241	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
242
243	/* don't need to lock because this is the last use of the ifq */
244
245	ifq->ifq_ops->ifqop_purge(ifq, &ml);
246	ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q);
247
248	ml_purge(&ml);
249}
250
251void
252ifq_add_data(struct ifqueue *ifq, struct if_data *data)
253{
254	mtx_enter(&ifq->ifq_mtx);
255	data->ifi_opackets += ifq->ifq_packets;
256	data->ifi_obytes += ifq->ifq_bytes;
257	data->ifi_oqdrops += ifq->ifq_qdrops;
258	data->ifi_omcasts += ifq->ifq_mcasts;
259	/* ifp->if_data.ifi_oerrors */
260	mtx_leave(&ifq->ifq_mtx);
261}
262
263int
264ifq_enqueue(struct ifqueue *ifq, struct mbuf *m)
265{
266	struct mbuf *dm;
267
268	mtx_enter(&ifq->ifq_mtx);
269	dm = ifq->ifq_ops->ifqop_enq(ifq, m);
270	if (dm != m) {
271		ifq->ifq_packets++;
272		ifq->ifq_bytes += m->m_pkthdr.len;
273		if (ISSET(m->m_flags, M_MCAST))
274			ifq->ifq_mcasts++;
275	}
276
277	if (dm == NULL)
278		ifq->ifq_len++;
279	else
280		ifq->ifq_qdrops++;
281	mtx_leave(&ifq->ifq_mtx);
282
283	if (dm != NULL)
284		m_freem(dm);
285
286	return (dm == m ? ENOBUFS : 0);
287}
288
289static inline void
290ifq_deq_enter(struct ifqueue *ifq)
291{
292	mtx_enter(&ifq->ifq_mtx);
293}
294
295static inline void
296ifq_deq_leave(struct ifqueue *ifq)
297{
298	struct mbuf_list ml;
299
300	ml = ifq->ifq_free;
301	ml_init(&ifq->ifq_free);
302
303	mtx_leave(&ifq->ifq_mtx);
304
305	if (!ml_empty(&ml))
306		ml_purge(&ml);
307}
308
309struct mbuf *
310ifq_deq_begin(struct ifqueue *ifq)
311{
312	struct mbuf *m = NULL;
313	void *cookie;
314
315	ifq_deq_enter(ifq);
316	if (ifq->ifq_len == 0 ||
317	    (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) {
318		ifq_deq_leave(ifq);
319		return (NULL);
320	}
321
322	m->m_pkthdr.ph_cookie = cookie;
323
324	return (m);
325}
326
327void
328ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m)
329{
330	void *cookie;
331
332	KASSERT(m != NULL);
333	cookie = m->m_pkthdr.ph_cookie;
334
335	ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie);
336	ifq->ifq_len--;
337	ifq_deq_leave(ifq);
338}
339
340void
341ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m)
342{
343	KASSERT(m != NULL);
344
345	ifq_deq_leave(ifq);
346}
347
348struct mbuf *
349ifq_dequeue(struct ifqueue *ifq)
350{
351	struct mbuf *m;
352
353	m = ifq_deq_begin(ifq);
354	if (m == NULL)
355		return (NULL);
356
357	ifq_deq_commit(ifq, m);
358
359	return (m);
360}
361
362unsigned int
363ifq_purge(struct ifqueue *ifq)
364{
365	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
366	unsigned int rv;
367
368	mtx_enter(&ifq->ifq_mtx);
369	ifq->ifq_ops->ifqop_purge(ifq, &ml);
370	rv = ifq->ifq_len;
371	ifq->ifq_len = 0;
372	ifq->ifq_qdrops += rv;
373	mtx_leave(&ifq->ifq_mtx);
374
375	KASSERT(rv == ml_len(&ml));
376
377	ml_purge(&ml);
378
379	return (rv);
380}
381
382void *
383ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops)
384{
385	mtx_enter(&ifq->ifq_mtx);
386	if (ifq->ifq_ops == ops)
387		return (ifq->ifq_q);
388
389	mtx_leave(&ifq->ifq_mtx);
390
391	return (NULL);
392}
393
394void
395ifq_q_leave(struct ifqueue *ifq, void *q)
396{
397	KASSERT(q == ifq->ifq_q);
398	mtx_leave(&ifq->ifq_mtx);
399}
400
401void
402ifq_mfreem(struct ifqueue *ifq, struct mbuf *m)
403{
404	MUTEX_ASSERT_LOCKED(&ifq->ifq_mtx);
405
406	ifq->ifq_len--;
407	ifq->ifq_qdrops++;
408	ml_enqueue(&ifq->ifq_free, m);
409}
410
411void
412ifq_mfreeml(struct ifqueue *ifq, struct mbuf_list *ml)
413{
414	MUTEX_ASSERT_LOCKED(&ifq->ifq_mtx);
415
416	ifq->ifq_len -= ml_len(ml);
417	ifq->ifq_qdrops += ml_len(ml);
418	ml_enlist(&ifq->ifq_free, ml);
419}
420
421/*
422 * priq implementation
423 */
424
425unsigned int
426priq_idx(unsigned int nqueues, const struct mbuf *m)
427{
428	unsigned int flow = 0;
429
430	if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
431		flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
432
433	return (flow % nqueues);
434}
435
436void *
437priq_alloc(unsigned int idx, void *null)
438{
439	struct priq *pq;
440	int i;
441
442	pq = malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK);
443	for (i = 0; i < IFQ_NQUEUES; i++)
444		ml_init(&pq->pq_lists[i]);
445	return (pq);
446}
447
448void
449priq_free(unsigned int idx, void *pq)
450{
451	free(pq, M_DEVBUF, sizeof(struct priq));
452}
453
454struct mbuf *
455priq_enq(struct ifqueue *ifq, struct mbuf *m)
456{
457	struct priq *pq;
458	struct mbuf_list *pl;
459	struct mbuf *n = NULL;
460	unsigned int prio;
461
462	pq = ifq->ifq_q;
463	KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO);
464
465	/* Find a lower priority queue to drop from */
466	if (ifq_len(ifq) >= ifq->ifq_maxlen) {
467		for (prio = 0; prio < m->m_pkthdr.pf.prio; prio++) {
468			pl = &pq->pq_lists[prio];
469			if (ml_len(pl) > 0) {
470				n = ml_dequeue(pl);
471				goto enqueue;
472			}
473		}
474		/*
475		 * There's no lower priority queue that we can
476		 * drop from so don't enqueue this one.
477		 */
478		return (m);
479	}
480
481 enqueue:
482	pl = &pq->pq_lists[m->m_pkthdr.pf.prio];
483	ml_enqueue(pl, m);
484
485	return (n);
486}
487
488struct mbuf *
489priq_deq_begin(struct ifqueue *ifq, void **cookiep)
490{
491	struct priq *pq = ifq->ifq_q;
492	struct mbuf_list *pl;
493	unsigned int prio = nitems(pq->pq_lists);
494	struct mbuf *m;
495
496	do {
497		pl = &pq->pq_lists[--prio];
498		m = MBUF_LIST_FIRST(pl);
499		if (m != NULL) {
500			*cookiep = pl;
501			return (m);
502		}
503	} while (prio > 0);
504
505	return (NULL);
506}
507
508void
509priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie)
510{
511	struct mbuf_list *pl = cookie;
512
513	KASSERT(MBUF_LIST_FIRST(pl) == m);
514
515	ml_dequeue(pl);
516}
517
518void
519priq_purge(struct ifqueue *ifq, struct mbuf_list *ml)
520{
521	struct priq *pq = ifq->ifq_q;
522	struct mbuf_list *pl;
523	unsigned int prio = nitems(pq->pq_lists);
524
525	do {
526		pl = &pq->pq_lists[--prio];
527		ml_enlist(ml, pl);
528	} while (prio > 0);
529}
530