ifq.c revision 1.46 
1/*	$OpenBSD: ifq.c,v 1.46 2022/04/30 21:13:57 bluhm Exp $ */
2
3/*
4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include "bpfilter.h"
20#include "kstat.h"
21
22#include <sys/param.h>
23#include <sys/systm.h>
24#include <sys/socket.h>
25#include <sys/mbuf.h>
26#include <sys/proc.h>
27#include <sys/sysctl.h>
28
29#include <net/if.h>
30#include <net/if_var.h>
31
32#if NBPFILTER > 0
33#include <net/bpf.h>
34#endif
35
36#if NKSTAT > 0
37#include <sys/kstat.h>
38#endif
39
40/*
41 * priq glue
42 */
43unsigned int	 priq_idx(unsigned int, const struct mbuf *);
44struct mbuf	*priq_enq(struct ifqueue *, struct mbuf *);
45struct mbuf	*priq_deq_begin(struct ifqueue *, void **);
46void		 priq_deq_commit(struct ifqueue *, struct mbuf *, void *);
47void		 priq_purge(struct ifqueue *, struct mbuf_list *);
48
49void		*priq_alloc(unsigned int, void *);
50void		 priq_free(unsigned int, void *);
51
52const struct ifq_ops priq_ops = {
53	priq_idx,
54	priq_enq,
55	priq_deq_begin,
56	priq_deq_commit,
57	priq_purge,
58	priq_alloc,
59	priq_free,
60};
61
62const struct ifq_ops * const ifq_priq_ops = &priq_ops;
63
64/*
65 * priq internal structures
66 */
67
68struct priq {
69	struct mbuf_list	 pq_lists[IFQ_NQUEUES];
70};
71
72/*
73 * ifqueue serialiser
74 */
75
76void	ifq_start_task(void *);
77void	ifq_restart_task(void *);
78void	ifq_barrier_task(void *);
79void	ifq_bundle_task(void *);
80
81static inline void
82ifq_run_start(struct ifqueue *ifq)
83{
84	ifq_serialize(ifq, &ifq->ifq_start);
85}
86
87void
88ifq_serialize(struct ifqueue *ifq, struct task *t)
89{
90	struct task work;
91
92	if (ISSET(t->t_flags, TASK_ONQUEUE))
93		return;
94
95	mtx_enter(&ifq->ifq_task_mtx);
96	if (!ISSET(t->t_flags, TASK_ONQUEUE)) {
97		SET(t->t_flags, TASK_ONQUEUE);
98		TAILQ_INSERT_TAIL(&ifq->ifq_task_list, t, t_entry);
99	}
100
101	if (ifq->ifq_serializer == NULL) {
102		ifq->ifq_serializer = curcpu();
103
104		while ((t = TAILQ_FIRST(&ifq->ifq_task_list)) != NULL) {
105			TAILQ_REMOVE(&ifq->ifq_task_list, t, t_entry);
106			CLR(t->t_flags, TASK_ONQUEUE);
107			work = *t; /* copy to caller to avoid races */
108
109			mtx_leave(&ifq->ifq_task_mtx);
110
111			(*work.t_func)(work.t_arg);
112
113			mtx_enter(&ifq->ifq_task_mtx);
114		}
115
116		ifq->ifq_serializer = NULL;
117	}
118	mtx_leave(&ifq->ifq_task_mtx);
119}
120
121int
122ifq_is_serialized(struct ifqueue *ifq)
123{
124	return (ifq->ifq_serializer == curcpu());
125}
126
127void
128ifq_start(struct ifqueue *ifq)
129{
130	if (ifq_len(ifq) >= min(ifq->ifq_if->if_txmit, ifq->ifq_maxlen)) {
131		task_del(ifq->ifq_softnet, &ifq->ifq_bundle);
132		ifq_run_start(ifq);
133	} else
134		task_add(ifq->ifq_softnet, &ifq->ifq_bundle);
135}
136
137void
138ifq_start_task(void *p)
139{
140	struct ifqueue *ifq = p;
141	struct ifnet *ifp = ifq->ifq_if;
142
143	if (!ISSET(ifp->if_flags, IFF_RUNNING) ||
144	    ifq_empty(ifq) || ifq_is_oactive(ifq))
145		return;
146
147	ifp->if_qstart(ifq);
148}
149
150void
151ifq_restart_task(void *p)
152{
153	struct ifqueue *ifq = p;
154	struct ifnet *ifp = ifq->ifq_if;
155
156	ifq_clr_oactive(ifq);
157	ifp->if_qstart(ifq);
158}
159
160void
161ifq_bundle_task(void *p)
162{
163	struct ifqueue *ifq = p;
164
165	ifq_run_start(ifq);
166}
167
168void
169ifq_barrier(struct ifqueue *ifq)
170{
171	struct cond c = COND_INITIALIZER();
172	struct task t = TASK_INITIALIZER(ifq_barrier_task, &c);
173
174	task_del(ifq->ifq_softnet, &ifq->ifq_bundle);
175
176	if (ifq->ifq_serializer == NULL)
177		return;
178
179	ifq_serialize(ifq, &t);
180
181	cond_wait(&c, "ifqbar");
182}
183
184void
185ifq_barrier_task(void *p)
186{
187	struct cond *c = p;
188
189	cond_signal(c);
190}
191
192/*
193 * ifqueue mbuf queue API
194 */
195
196#if NKSTAT > 0
197struct ifq_kstat_data {
198	struct kstat_kv kd_packets;
199	struct kstat_kv kd_bytes;
200	struct kstat_kv kd_qdrops;
201	struct kstat_kv kd_errors;
202	struct kstat_kv kd_qlen;
203	struct kstat_kv kd_maxqlen;
204	struct kstat_kv kd_oactive;
205};
206
207static const struct ifq_kstat_data ifq_kstat_tpl = {
208	KSTAT_KV_UNIT_INITIALIZER("packets",
209	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
210	KSTAT_KV_UNIT_INITIALIZER("bytes",
211	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES),
212	KSTAT_KV_UNIT_INITIALIZER("qdrops",
213	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
214	KSTAT_KV_UNIT_INITIALIZER("errors",
215	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
216	KSTAT_KV_UNIT_INITIALIZER("qlen",
217	    KSTAT_KV_T_UINT32, KSTAT_KV_U_PACKETS),
218	KSTAT_KV_UNIT_INITIALIZER("maxqlen",
219	    KSTAT_KV_T_UINT32, KSTAT_KV_U_PACKETS),
220	KSTAT_KV_INITIALIZER("oactive", KSTAT_KV_T_BOOL),
221};
222
223int
224ifq_kstat_copy(struct kstat *ks, void *dst)
225{
226	struct ifqueue *ifq = ks->ks_softc;
227	struct ifq_kstat_data *kd = dst;
228
229	*kd = ifq_kstat_tpl;
230	kstat_kv_u64(&kd->kd_packets) = ifq->ifq_packets;
231	kstat_kv_u64(&kd->kd_bytes) = ifq->ifq_bytes;
232	kstat_kv_u64(&kd->kd_qdrops) = ifq->ifq_qdrops;
233	kstat_kv_u64(&kd->kd_errors) = ifq->ifq_errors;
234	kstat_kv_u32(&kd->kd_qlen) = ifq->ifq_len;
235	kstat_kv_u32(&kd->kd_maxqlen) = ifq->ifq_maxlen;
236	kstat_kv_bool(&kd->kd_oactive) = ifq->ifq_oactive;
237
238	return (0);
239}
240#endif
241
242void
243ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
244{
245	ifq->ifq_if = ifp;
246	ifq->ifq_softnet = net_tq(ifp->if_index + idx);
247	ifq->ifq_softc = NULL;
248
249	mtx_init(&ifq->ifq_mtx, IPL_NET);
250
251	/* default to priq */
252	ifq->ifq_ops = &priq_ops;
253	ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL);
254
255	ml_init(&ifq->ifq_free);
256	ifq->ifq_len = 0;
257
258	ifq->ifq_packets = 0;
259	ifq->ifq_bytes = 0;
260	ifq->ifq_qdrops = 0;
261	ifq->ifq_errors = 0;
262	ifq->ifq_mcasts = 0;
263
264	mtx_init(&ifq->ifq_task_mtx, IPL_NET);
265	TAILQ_INIT(&ifq->ifq_task_list);
266	ifq->ifq_serializer = NULL;
267	task_set(&ifq->ifq_bundle, ifq_bundle_task, ifq);
268
269	task_set(&ifq->ifq_start, ifq_start_task, ifq);
270	task_set(&ifq->ifq_restart, ifq_restart_task, ifq);
271
272	if (ifq->ifq_maxlen == 0)
273		ifq_set_maxlen(ifq, IFQ_MAXLEN);
274
275	ifq->ifq_idx = idx;
276
277#if NKSTAT > 0
278	/* XXX xname vs driver name and unit */
279	ifq->ifq_kstat = kstat_create(ifp->if_xname, 0,
280	    "txq", ifq->ifq_idx, KSTAT_T_KV, 0);
281	KASSERT(ifq->ifq_kstat != NULL);
282	kstat_set_mutex(ifq->ifq_kstat, &ifq->ifq_mtx);
283	ifq->ifq_kstat->ks_softc = ifq;
284	ifq->ifq_kstat->ks_datalen = sizeof(ifq_kstat_tpl);
285	ifq->ifq_kstat->ks_copy = ifq_kstat_copy;
286	kstat_install(ifq->ifq_kstat);
287#endif
288}
289
290void
291ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg)
292{
293	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
294	struct mbuf_list free_ml = MBUF_LIST_INITIALIZER();
295	struct mbuf *m;
296	const struct ifq_ops *oldops;
297	void *newq, *oldq;
298
299	newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg);
300
301	mtx_enter(&ifq->ifq_mtx);
302	ifq->ifq_ops->ifqop_purge(ifq, &ml);
303	ifq->ifq_len = 0;
304
305	oldops = ifq->ifq_ops;
306	oldq = ifq->ifq_q;
307
308	ifq->ifq_ops = newops;
309	ifq->ifq_q = newq;
310
311	while ((m = ml_dequeue(&ml)) != NULL) {
312		m = ifq->ifq_ops->ifqop_enq(ifq, m);
313		if (m != NULL) {
314			ifq->ifq_qdrops++;
315			ml_enqueue(&free_ml, m);
316		} else
317			ifq->ifq_len++;
318	}
319	mtx_leave(&ifq->ifq_mtx);
320
321	oldops->ifqop_free(ifq->ifq_idx, oldq);
322
323	ml_purge(&free_ml);
324}
325
326void
327ifq_destroy(struct ifqueue *ifq)
328{
329	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
330
331#if NKSTAT > 0
332	kstat_destroy(ifq->ifq_kstat);
333#endif
334
335	NET_ASSERT_UNLOCKED();
336	if (!task_del(ifq->ifq_softnet, &ifq->ifq_bundle))
337		taskq_barrier(ifq->ifq_softnet);
338
339	/* don't need to lock because this is the last use of the ifq */
340
341	ifq->ifq_ops->ifqop_purge(ifq, &ml);
342	ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q);
343
344	ml_purge(&ml);
345}
346
347void
348ifq_add_data(struct ifqueue *ifq, struct if_data *data)
349{
350	mtx_enter(&ifq->ifq_mtx);
351	data->ifi_opackets += ifq->ifq_packets;
352	data->ifi_obytes += ifq->ifq_bytes;
353	data->ifi_oqdrops += ifq->ifq_qdrops;
354	data->ifi_omcasts += ifq->ifq_mcasts;
355	/* ifp->if_data.ifi_oerrors */
356	mtx_leave(&ifq->ifq_mtx);
357}
358
359int
360ifq_enqueue(struct ifqueue *ifq, struct mbuf *m)
361{
362	struct mbuf *dm;
363
364	mtx_enter(&ifq->ifq_mtx);
365	dm = ifq->ifq_ops->ifqop_enq(ifq, m);
366	if (dm != m) {
367		ifq->ifq_packets++;
368		ifq->ifq_bytes += m->m_pkthdr.len;
369		if (ISSET(m->m_flags, M_MCAST))
370			ifq->ifq_mcasts++;
371	}
372
373	if (dm == NULL)
374		ifq->ifq_len++;
375	else
376		ifq->ifq_qdrops++;
377	mtx_leave(&ifq->ifq_mtx);
378
379	if (dm != NULL)
380		m_freem(dm);
381
382	return (dm == m ? ENOBUFS : 0);
383}
384
385static inline void
386ifq_deq_enter(struct ifqueue *ifq)
387{
388	mtx_enter(&ifq->ifq_mtx);
389}
390
391static inline void
392ifq_deq_leave(struct ifqueue *ifq)
393{
394	struct mbuf_list ml;
395
396	ml = ifq->ifq_free;
397	ml_init(&ifq->ifq_free);
398
399	mtx_leave(&ifq->ifq_mtx);
400
401	if (!ml_empty(&ml))
402		ml_purge(&ml);
403}
404
405struct mbuf *
406ifq_deq_begin(struct ifqueue *ifq)
407{
408	struct mbuf *m = NULL;
409	void *cookie;
410
411	ifq_deq_enter(ifq);
412	if (ifq->ifq_len == 0 ||
413	    (m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie)) == NULL) {
414		ifq_deq_leave(ifq);
415		return (NULL);
416	}
417
418	m->m_pkthdr.ph_cookie = cookie;
419
420	return (m);
421}
422
423void
424ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m)
425{
426	void *cookie;
427
428	KASSERT(m != NULL);
429	cookie = m->m_pkthdr.ph_cookie;
430
431	ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie);
432	ifq->ifq_len--;
433	ifq_deq_leave(ifq);
434}
435
436void
437ifq_deq_rollback(struct ifqueue *ifq, struct mbuf *m)
438{
439	KASSERT(m != NULL);
440
441	ifq_deq_leave(ifq);
442}
443
444struct mbuf *
445ifq_dequeue(struct ifqueue *ifq)
446{
447	struct mbuf *m;
448
449	m = ifq_deq_begin(ifq);
450	if (m == NULL)
451		return (NULL);
452
453	ifq_deq_commit(ifq, m);
454
455	return (m);
456}
457
458int
459ifq_deq_sleep(struct ifqueue *ifq, struct mbuf **mp, int nbio, int priority,
460    const char *wmesg, volatile unsigned int *sleeping,
461    volatile unsigned int *alive)
462{
463	struct mbuf *m;
464	void *cookie;
465	int error = 0;
466
467	ifq_deq_enter(ifq);
468	if (ifq->ifq_len == 0 && nbio)
469		error = EWOULDBLOCK;
470	else {
471		for (;;) {
472			m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie);
473			if (m != NULL) {
474				ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie);
475				ifq->ifq_len--;
476				*mp = m;
477				break;
478			}
479
480			(*sleeping)++;
481			error = msleep_nsec(ifq, &ifq->ifq_mtx,
482			    priority, wmesg, INFSLP);
483			(*sleeping)--;
484			if (error != 0)
485				break;
486			if (!(*alive)) {
487				error = EIO;
488				break;
489			}
490		}
491	}
492	ifq_deq_leave(ifq);
493
494	return (error);
495}
496
497int
498ifq_hdatalen(struct ifqueue *ifq)
499{
500	struct mbuf *m;
501	int len = 0;
502
503	if (ifq_empty(ifq))
504		return (0);
505
506	m = ifq_deq_begin(ifq);
507	if (m != NULL) {
508		len = m->m_pkthdr.len;
509		ifq_deq_rollback(ifq, m);
510	}
511
512	return (len);
513}
514
515unsigned int
516ifq_purge(struct ifqueue *ifq)
517{
518	struct mbuf_list ml = MBUF_LIST_INITIALIZER();
519	unsigned int rv;
520
521	mtx_enter(&ifq->ifq_mtx);
522	ifq->ifq_ops->ifqop_purge(ifq, &ml);
523	rv = ifq->ifq_len;
524	ifq->ifq_len = 0;
525	ifq->ifq_qdrops += rv;
526	mtx_leave(&ifq->ifq_mtx);
527
528	KASSERT(rv == ml_len(&ml));
529
530	ml_purge(&ml);
531
532	return (rv);
533}
534
535void *
536ifq_q_enter(struct ifqueue *ifq, const struct ifq_ops *ops)
537{
538	mtx_enter(&ifq->ifq_mtx);
539	if (ifq->ifq_ops == ops)
540		return (ifq->ifq_q);
541
542	mtx_leave(&ifq->ifq_mtx);
543
544	return (NULL);
545}
546
547void
548ifq_q_leave(struct ifqueue *ifq, void *q)
549{
550	KASSERT(q == ifq->ifq_q);
551	mtx_leave(&ifq->ifq_mtx);
552}
553
554void
555ifq_mfreem(struct ifqueue *ifq, struct mbuf *m)
556{
557	MUTEX_ASSERT_LOCKED(&ifq->ifq_mtx);
558
559	ifq->ifq_len--;
560	ifq->ifq_qdrops++;
561	ml_enqueue(&ifq->ifq_free, m);
562}
563
564void
565ifq_mfreeml(struct ifqueue *ifq, struct mbuf_list *ml)
566{
567	MUTEX_ASSERT_LOCKED(&ifq->ifq_mtx);
568
569	ifq->ifq_len -= ml_len(ml);
570	ifq->ifq_qdrops += ml_len(ml);
571	ml_enlist(&ifq->ifq_free, ml);
572}
573
574/*
575 * ifiq
576 */
577
578#if NKSTAT > 0
579struct ifiq_kstat_data {
580	struct kstat_kv kd_packets;
581	struct kstat_kv kd_bytes;
582	struct kstat_kv kd_qdrops;
583	struct kstat_kv kd_errors;
584	struct kstat_kv kd_qlen;
585};
586
587static const struct ifiq_kstat_data ifiq_kstat_tpl = {
588	KSTAT_KV_UNIT_INITIALIZER("packets",
589	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
590	KSTAT_KV_UNIT_INITIALIZER("bytes",
591	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES),
592	KSTAT_KV_UNIT_INITIALIZER("qdrops",
593	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
594	KSTAT_KV_UNIT_INITIALIZER("errors",
595	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
596	KSTAT_KV_UNIT_INITIALIZER("qlen",
597	    KSTAT_KV_T_UINT32, KSTAT_KV_U_PACKETS),
598};
599
600int
601ifiq_kstat_copy(struct kstat *ks, void *dst)
602{
603	struct ifiqueue *ifiq = ks->ks_softc;
604	struct ifiq_kstat_data *kd = dst;
605
606	*kd = ifiq_kstat_tpl;
607	kstat_kv_u64(&kd->kd_packets) = ifiq->ifiq_packets;
608	kstat_kv_u64(&kd->kd_bytes) = ifiq->ifiq_bytes;
609	kstat_kv_u64(&kd->kd_qdrops) = ifiq->ifiq_qdrops;
610	kstat_kv_u64(&kd->kd_errors) = ifiq->ifiq_errors;
611	kstat_kv_u32(&kd->kd_qlen) = ml_len(&ifiq->ifiq_ml);
612
613	return (0);
614}
615#endif
616
617static void	ifiq_process(void *);
618
619void
620ifiq_init(struct ifiqueue *ifiq, struct ifnet *ifp, unsigned int idx)
621{
622	ifiq->ifiq_if = ifp;
623	ifiq->ifiq_softnet = net_tq(ifp->if_index + idx);
624	ifiq->ifiq_softc = NULL;
625
626	mtx_init(&ifiq->ifiq_mtx, IPL_NET);
627	ml_init(&ifiq->ifiq_ml);
628	task_set(&ifiq->ifiq_task, ifiq_process, ifiq);
629	ifiq->ifiq_pressure = 0;
630
631	ifiq->ifiq_packets = 0;
632	ifiq->ifiq_bytes = 0;
633	ifiq->ifiq_qdrops = 0;
634	ifiq->ifiq_errors = 0;
635
636	ifiq->ifiq_idx = idx;
637
638#if NKSTAT > 0
639	/* XXX xname vs driver name and unit */
640	ifiq->ifiq_kstat = kstat_create(ifp->if_xname, 0,
641	    "rxq", ifiq->ifiq_idx, KSTAT_T_KV, 0);
642	KASSERT(ifiq->ifiq_kstat != NULL);
643	kstat_set_mutex(ifiq->ifiq_kstat, &ifiq->ifiq_mtx);
644	ifiq->ifiq_kstat->ks_softc = ifiq;
645	ifiq->ifiq_kstat->ks_datalen = sizeof(ifiq_kstat_tpl);
646	ifiq->ifiq_kstat->ks_copy = ifiq_kstat_copy;
647	kstat_install(ifiq->ifiq_kstat);
648#endif
649}
650
651void
652ifiq_destroy(struct ifiqueue *ifiq)
653{
654#if NKSTAT > 0
655	kstat_destroy(ifiq->ifiq_kstat);
656#endif
657
658	NET_ASSERT_UNLOCKED();
659	if (!task_del(ifiq->ifiq_softnet, &ifiq->ifiq_task))
660		taskq_barrier(ifiq->ifiq_softnet);
661
662	/* don't need to lock because this is the last use of the ifiq */
663	ml_purge(&ifiq->ifiq_ml);
664}
665
666unsigned int ifiq_maxlen_drop = 2048 * 5;
667unsigned int ifiq_maxlen_return = 2048 * 3;
668
669int
670ifiq_input(struct ifiqueue *ifiq, struct mbuf_list *ml)
671{
672	struct ifnet *ifp = ifiq->ifiq_if;
673	struct mbuf *m;
674	uint64_t packets;
675	uint64_t bytes = 0;
676	unsigned int len;
677#if NBPFILTER > 0
678	caddr_t if_bpf;
679#endif
680
681	if (ml_empty(ml))
682		return (0);
683
684	MBUF_LIST_FOREACH(ml, m) {
685		m->m_pkthdr.ph_ifidx = ifp->if_index;
686		m->m_pkthdr.ph_rtableid = ifp->if_rdomain;
687		bytes += m->m_pkthdr.len;
688	}
689	packets = ml_len(ml);
690
691#if NBPFILTER > 0
692	if_bpf = ifp->if_bpf;
693	if (if_bpf) {
694		struct mbuf_list ml0 = *ml;
695
696		ml_init(ml);
697
698		while ((m = ml_dequeue(&ml0)) != NULL) {
699			if ((*ifp->if_bpf_mtap)(if_bpf, m, BPF_DIRECTION_IN))
700				m_freem(m);
701			else
702				ml_enqueue(ml, m);
703		}
704
705		if (ml_empty(ml)) {
706			mtx_enter(&ifiq->ifiq_mtx);
707			ifiq->ifiq_packets += packets;
708			ifiq->ifiq_bytes += bytes;
709			mtx_leave(&ifiq->ifiq_mtx);
710
711			return (0);
712		}
713	}
714#endif
715
716	mtx_enter(&ifiq->ifiq_mtx);
717	ifiq->ifiq_packets += packets;
718	ifiq->ifiq_bytes += bytes;
719
720	len = ml_len(&ifiq->ifiq_ml);
721	if (__predict_true(!ISSET(ifp->if_xflags, IFXF_MONITOR))) {
722		if (len > ifiq_maxlen_drop)
723			ifiq->ifiq_qdrops += ml_len(ml);
724		else
725			ml_enlist(&ifiq->ifiq_ml, ml);
726	}
727	mtx_leave(&ifiq->ifiq_mtx);
728
729	if (ml_empty(ml))
730		task_add(ifiq->ifiq_softnet, &ifiq->ifiq_task);
731	else
732		ml_purge(ml);
733
734	return (len > ifiq_maxlen_return);
735}
736
737void
738ifiq_add_data(struct ifiqueue *ifiq, struct if_data *data)
739{
740	mtx_enter(&ifiq->ifiq_mtx);
741	data->ifi_ipackets += ifiq->ifiq_packets;
742	data->ifi_ibytes += ifiq->ifiq_bytes;
743	data->ifi_iqdrops += ifiq->ifiq_qdrops;
744	mtx_leave(&ifiq->ifiq_mtx);
745}
746
747int
748ifiq_enqueue(struct ifiqueue *ifiq, struct mbuf *m)
749{
750	mtx_enter(&ifiq->ifiq_mtx);
751	ml_enqueue(&ifiq->ifiq_ml, m);
752	mtx_leave(&ifiq->ifiq_mtx);
753
754	task_add(ifiq->ifiq_softnet, &ifiq->ifiq_task);
755
756	return (0);
757}
758
759static void
760ifiq_process(void *arg)
761{
762	struct ifiqueue *ifiq = arg;
763	struct mbuf_list ml;
764
765	if (ifiq_empty(ifiq))
766		return;
767
768	mtx_enter(&ifiq->ifiq_mtx);
769	ml = ifiq->ifiq_ml;
770	ml_init(&ifiq->ifiq_ml);
771	mtx_leave(&ifiq->ifiq_mtx);
772
773	if_input_process(ifiq->ifiq_if, &ml);
774}
775
776int
777net_ifiq_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp,
778    void *newp, size_t newlen)
779{
780	int error = EOPNOTSUPP;
781/* pressure is disabled for 6.6-release */
782#if 0
783	int val;
784
785	if (namelen != 1)
786		return (EISDIR);
787
788	switch (name[0]) {
789	case NET_LINK_IFRXQ_PRESSURE_RETURN:
790		val = ifiq_pressure_return;
791		error = sysctl_int(oldp, oldlenp, newp, newlen, &val);
792		if (error != 0)
793			return (error);
794		if (val < 1 || val > ifiq_pressure_drop)
795			return (EINVAL);
796		ifiq_pressure_return = val;
797		break;
798	case NET_LINK_IFRXQ_PRESSURE_DROP:
799		val = ifiq_pressure_drop;
800		error = sysctl_int(oldp, oldlenp, newp, newlen, &val);
801		if (error != 0)
802			return (error);
803		if (ifiq_pressure_return > val)
804			return (EINVAL);
805		ifiq_pressure_drop = val;
806		break;
807	default:
808		error = EOPNOTSUPP;
809		break;
810	}
811#endif
812
813	return (error);
814}
815
816/*
817 * priq implementation
818 */
819
820unsigned int
821priq_idx(unsigned int nqueues, const struct mbuf *m)
822{
823	unsigned int flow = 0;
824
825	if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
826		flow = m->m_pkthdr.ph_flowid;
827
828	return (flow % nqueues);
829}
830
831void *
832priq_alloc(unsigned int idx, void *null)
833{
834	struct priq *pq;
835	int i;
836
837	pq = malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK);
838	for (i = 0; i < IFQ_NQUEUES; i++)
839		ml_init(&pq->pq_lists[i]);
840	return (pq);
841}
842
843void
844priq_free(unsigned int idx, void *pq)
845{
846	free(pq, M_DEVBUF, sizeof(struct priq));
847}
848
849struct mbuf *
850priq_enq(struct ifqueue *ifq, struct mbuf *m)
851{
852	struct priq *pq;
853	struct mbuf_list *pl;
854	struct mbuf *n = NULL;
855	unsigned int prio;
856
857	pq = ifq->ifq_q;
858	KASSERT(m->m_pkthdr.pf.prio <= IFQ_MAXPRIO);
859
860	/* Find a lower priority queue to drop from */
861	if (ifq_len(ifq) >= ifq->ifq_maxlen) {
862		for (prio = 0; prio < m->m_pkthdr.pf.prio; prio++) {
863			pl = &pq->pq_lists[prio];
864			if (ml_len(pl) > 0) {
865				n = ml_dequeue(pl);
866				goto enqueue;
867			}
868		}
869		/*
870		 * There's no lower priority queue that we can
871		 * drop from so don't enqueue this one.
872		 */
873		return (m);
874	}
875
876 enqueue:
877	pl = &pq->pq_lists[m->m_pkthdr.pf.prio];
878	ml_enqueue(pl, m);
879
880	return (n);
881}
882
883struct mbuf *
884priq_deq_begin(struct ifqueue *ifq, void **cookiep)
885{
886	struct priq *pq = ifq->ifq_q;
887	struct mbuf_list *pl;
888	unsigned int prio = nitems(pq->pq_lists);
889	struct mbuf *m;
890
891	do {
892		pl = &pq->pq_lists[--prio];
893		m = MBUF_LIST_FIRST(pl);
894		if (m != NULL) {
895			*cookiep = pl;
896			return (m);
897		}
898	} while (prio > 0);
899
900	return (NULL);
901}
902
903void
904priq_deq_commit(struct ifqueue *ifq, struct mbuf *m, void *cookie)
905{
906	struct mbuf_list *pl = cookie;
907
908	KASSERT(MBUF_LIST_FIRST(pl) == m);
909
910	ml_dequeue(pl);
911}
912
913void
914priq_purge(struct ifqueue *ifq, struct mbuf_list *ml)
915{
916	struct priq *pq = ifq->ifq_q;
917	struct mbuf_list *pl;
918	unsigned int prio = nitems(pq->pq_lists);
919
920	do {
921		pl = &pq->pq_lists[--prio];
922		ml_enlist(ml, pl);
923	} while (prio > 0);
924}
925