Deleted Added
sdiff udiff text old ( 122152 ) new ( 122320 )
full compact
netisr.c (122152) netisr.c (122320)
1/*-
2 * Copyright (c) 2001,2002,2003 Jonathan Lemon <jlemon@FreeBSD.org>
3 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 10 unchanged lines hidden (view full) ---

19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
1/*-
2 * Copyright (c) 2001,2002,2003 Jonathan Lemon <jlemon@FreeBSD.org>
3 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 10 unchanged lines hidden (view full) ---

19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 * $FreeBSD: head/sys/net/netisr.c 122152 2003-11-05 23:42:51Z sam $
27 * $FreeBSD: head/sys/net/netisr.c 122320 2003-11-08 22:28:40Z sam $
28 */
29
30#include <sys/param.h>
31#include <sys/bus.h>
32#include <sys/rtprio.h>
33#include <sys/systm.h>
34#include <sys/interrupt.h>
35#include <sys/kernel.h>

--- 27 unchanged lines hidden (view full) ---

63SYSCTL_INT(_debug, OID_AUTO, mpsafenet, CTLFLAG_RD, &debug_mpsafenet, 0,
64 "Enable/disable MPSAFE network support");
65
66volatile unsigned int netisr; /* scheduling bits for network */
67
68struct netisr {
69 netisr_t *ni_handler;
70 struct ifqueue *ni_queue;
28 */
29
30#include <sys/param.h>
31#include <sys/bus.h>
32#include <sys/rtprio.h>
33#include <sys/systm.h>
34#include <sys/interrupt.h>
35#include <sys/kernel.h>

--- 27 unchanged lines hidden (view full) ---

63SYSCTL_INT(_debug, OID_AUTO, mpsafenet, CTLFLAG_RD, &debug_mpsafenet, 0,
64 "Enable/disable MPSAFE network support");
65
66volatile unsigned int netisr; /* scheduling bits for network */
67
68struct netisr {
69 netisr_t *ni_handler;
70 struct ifqueue *ni_queue;
71 int ni_flags;
71} netisrs[32];
72
72} netisrs[32];
73
73static struct mtx netisr_mtx;
74static void *net_ih;
75
76void
77legacy_setsoftnet(void)
78{
79 swi_sched(net_ih, 0);
80}
81
82void
74static void *net_ih;
75
76void
77legacy_setsoftnet(void)
78{
79 swi_sched(net_ih, 0);
80}
81
82void
83netisr_register(int num, netisr_t *handler, struct ifqueue *inq)
83netisr_register(int num, netisr_t *handler, struct ifqueue *inq, int flags)
84{
85
86 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
87 ("bad isr %d", num));
88 netisrs[num].ni_handler = handler;
89 netisrs[num].ni_queue = inq;
84{
85
86 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
87 ("bad isr %d", num));
88 netisrs[num].ni_handler = handler;
89 netisrs[num].ni_queue = inq;
90 if ((flags & NETISR_MPSAFE) && !debug_mpsafenet)
91 flags &= ~NETISR_MPSAFE;
92 netisrs[num].ni_flags = flags;
90}
91
92void
93netisr_unregister(int num)
94{
95 struct netisr *ni;
93}
94
95void
96netisr_unregister(int num)
97{
98 struct netisr *ni;
96 int s;
97
98 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
99 ("bad isr %d", num));
100 ni = &netisrs[num];
101 ni->ni_handler = NULL;
99
100 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
101 ("bad isr %d", num));
102 ni = &netisrs[num];
103 ni->ni_handler = NULL;
102 if (ni->ni_queue != NULL) {
103 s = splimp();
104 if (ni->ni_queue != NULL)
104 IF_DRAIN(ni->ni_queue);
105 IF_DRAIN(ni->ni_queue);
105 splx(s);
106 }
107}
108
109struct isrstat {
110 int isrs_count; /* dispatch count */
106}
107
108struct isrstat {
109 int isrs_count; /* dispatch count */
111 int isrs_directed; /* ...successfully dispatched */
110 int isrs_directed; /* ...directly dispatched */
112 int isrs_deferred; /* ...queued instead */
113 int isrs_queued; /* intentionally queueued */
111 int isrs_deferred; /* ...queued instead */
112 int isrs_queued; /* intentionally queueued */
113 int isrs_drop; /* dropped 'cuz no handler */
114 int isrs_swi_count; /* swi_net handlers called */
115};
116static struct isrstat isrstat;
117
118SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr counters");
119
120static int netisr_enable = 0;
121SYSCTL_INT(_net_isr, OID_AUTO, enable, CTLFLAG_RW,
122 &netisr_enable, 0, "enable direct dispatch");
123TUNABLE_INT("net.isr.enable", &netisr_enable);
124
125SYSCTL_INT(_net_isr, OID_AUTO, count, CTLFLAG_RD,
126 &isrstat.isrs_count, 0, "");
127SYSCTL_INT(_net_isr, OID_AUTO, directed, CTLFLAG_RD,
128 &isrstat.isrs_directed, 0, "");
129SYSCTL_INT(_net_isr, OID_AUTO, deferred, CTLFLAG_RD,
130 &isrstat.isrs_deferred, 0, "");
131SYSCTL_INT(_net_isr, OID_AUTO, queued, CTLFLAG_RD,
132 &isrstat.isrs_queued, 0, "");
114 int isrs_swi_count; /* swi_net handlers called */
115};
116static struct isrstat isrstat;
117
118SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr counters");
119
120static int netisr_enable = 0;
121SYSCTL_INT(_net_isr, OID_AUTO, enable, CTLFLAG_RW,
122 &netisr_enable, 0, "enable direct dispatch");
123TUNABLE_INT("net.isr.enable", &netisr_enable);
124
125SYSCTL_INT(_net_isr, OID_AUTO, count, CTLFLAG_RD,
126 &isrstat.isrs_count, 0, "");
127SYSCTL_INT(_net_isr, OID_AUTO, directed, CTLFLAG_RD,
128 &isrstat.isrs_directed, 0, "");
129SYSCTL_INT(_net_isr, OID_AUTO, deferred, CTLFLAG_RD,
130 &isrstat.isrs_deferred, 0, "");
131SYSCTL_INT(_net_isr, OID_AUTO, queued, CTLFLAG_RD,
132 &isrstat.isrs_queued, 0, "");
133SYSCTL_INT(_net_isr, OID_AUTO, drop, CTLFLAG_RD,
134 &isrstat.isrs_drop, 0, "");
133SYSCTL_INT(_net_isr, OID_AUTO, swi_count, CTLFLAG_RD,
134 &isrstat.isrs_swi_count, 0, "");
135
136/*
137 * Process all packets currently present in a netisr queue. Used to
138 * drain an existing set of packets waiting for processing when we
139 * begin direct dispatch, to avoid processing packets out of order.
140 */

--- 7 unchanged lines hidden (view full) ---

148 if (m == NULL)
149 break;
150 ni->ni_handler(m);
151 }
152}
153
154/*
155 * Call the netisr directly instead of queueing the packet, if possible.
135SYSCTL_INT(_net_isr, OID_AUTO, swi_count, CTLFLAG_RD,
136 &isrstat.isrs_swi_count, 0, "");
137
138/*
139 * Process all packets currently present in a netisr queue. Used to
140 * drain an existing set of packets waiting for processing when we
141 * begin direct dispatch, to avoid processing packets out of order.
142 */

--- 7 unchanged lines hidden (view full) ---

150 if (m == NULL)
151 break;
152 ni->ni_handler(m);
153 }
154}
155
156/*
157 * Call the netisr directly instead of queueing the packet, if possible.
156 *
157 * Ideally, the permissibility of calling the routine would be determined
158 * by checking if splnet() was asserted at the time the device interrupt
159 * occurred; if so, this indicates that someone is in the network stack.
160 *
161 * However, bus_setup_intr uses INTR_TYPE_NET, which sets splnet before
162 * calling the interrupt handler, so the previous mask is unavailable.
163 * Approximate this by checking intr_nesting_level instead; if any SWI
164 * handlers are running, the packet is queued instead.
165 */
166void
167netisr_dispatch(int num, struct mbuf *m)
168{
169 struct netisr *ni;
170
158 */
159void
160netisr_dispatch(int num, struct mbuf *m)
161{
162 struct netisr *ni;
163
171 isrstat.isrs_count++;
164 isrstat.isrs_count++; /* XXX redundant */
172 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
173 ("bad isr %d", num));
174 ni = &netisrs[num];
175 if (ni->ni_queue == NULL) {
165 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
166 ("bad isr %d", num));
167 ni = &netisrs[num];
168 if (ni->ni_queue == NULL) {
169 isrstat.isrs_drop++;
176 m_freem(m);
177 return;
178 }
170 m_freem(m);
171 return;
172 }
179 if (netisr_enable && mtx_trylock(&netisr_mtx)) {
173 /*
174 * Do direct dispatch only for MPSAFE netisrs (and
175 * only when enabled). Note that when a netisr is
176 * marked MPSAFE we permit multiple concurrent instances
177 * to run. We guarantee only the order in which
178 * packets are processed for each "dispatch point" in
179 * the system (i.e. call to netisr_dispatch or
180 * netisr_queue). This insures ordering of packets
181 * from an interface but does not guarantee ordering
182 * between multiple places in the system (e.g. IP
183 * dispatched from interfaces vs. IP queued from IPSec).
184 */
185 if (netisr_enable && (ni->ni_flags & NETISR_MPSAFE)) {
180 isrstat.isrs_directed++;
181 /*
186 isrstat.isrs_directed++;
187 /*
182 * One slight problem here is that packets might bypass
183 * each other in the stack, if an earlier one happened
184 * to get stuck in the queue.
185 *
186 * we can either:
187 * a. drain the queue before handling this packet,
188 * b. fallback to queueing the packet,
189 * c. sweep the issue under the rug and ignore it.
190 *
191 * Currently, we do a). Previously, we did c).
188 * NB: We used to drain the queue before handling
189 * the packet but now do not. Doing so here will
190 * not preserve ordering so instead we fallback to
191 * guaranteeing order only from dispatch points
192 * in the system (see above).
192 */
193 */
193 netisr_processqueue(ni);
194 ni->ni_handler(m);
194 ni->ni_handler(m);
195 mtx_unlock(&netisr_mtx);
196 } else {
197 isrstat.isrs_deferred++;
198 if (IF_HANDOFF(ni->ni_queue, m, NULL))
199 schednetisr(num);
200 }
201}
202
203/*

--- 5 unchanged lines hidden (view full) ---

209netisr_queue(int num, struct mbuf *m)
210{
211 struct netisr *ni;
212
213 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
214 ("bad isr %d", num));
215 ni = &netisrs[num];
216 if (ni->ni_queue == NULL) {
195 } else {
196 isrstat.isrs_deferred++;
197 if (IF_HANDOFF(ni->ni_queue, m, NULL))
198 schednetisr(num);
199 }
200}
201
202/*

--- 5 unchanged lines hidden (view full) ---

208netisr_queue(int num, struct mbuf *m)
209{
210 struct netisr *ni;
211
212 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))),
213 ("bad isr %d", num));
214 ni = &netisrs[num];
215 if (ni->ni_queue == NULL) {
216 isrstat.isrs_drop++;
217 m_freem(m);
218 return (1);
219 }
220 isrstat.isrs_queued++;
221 if (!IF_HANDOFF(ni->ni_queue, m, NULL))
222 return (0);
223 schednetisr(num);
224 return (1);

--- 6 unchanged lines hidden (view full) ---

231 u_int bits;
232 int i;
233#ifdef DEVICE_POLLING
234 const int polling = 1;
235#else
236 const int polling = 0;
237#endif
238
217 m_freem(m);
218 return (1);
219 }
220 isrstat.isrs_queued++;
221 if (!IF_HANDOFF(ni->ni_queue, m, NULL))
222 return (0);
223 schednetisr(num);
224 return (1);

--- 6 unchanged lines hidden (view full) ---

231 u_int bits;
232 int i;
233#ifdef DEVICE_POLLING
234 const int polling = 1;
235#else
236 const int polling = 0;
237#endif
238
239 mtx_lock(&netisr_mtx);
240 do {
241 bits = atomic_readandclear_int(&netisr);
242 if (bits == 0)
243 break;
244 while ((i = ffs(bits)) != 0) {
245 isrstat.isrs_swi_count++;
246 i--;
247 bits &= ~(1 << i);
248 ni = &netisrs[i];
249 if (ni->ni_handler == NULL) {
250 printf("swi_net: unregistered isr %d.\n", i);
251 continue;
252 }
239 do {
240 bits = atomic_readandclear_int(&netisr);
241 if (bits == 0)
242 break;
243 while ((i = ffs(bits)) != 0) {
244 isrstat.isrs_swi_count++;
245 i--;
246 bits &= ~(1 << i);
247 ni = &netisrs[i];
248 if (ni->ni_handler == NULL) {
249 printf("swi_net: unregistered isr %d.\n", i);
250 continue;
251 }
253 if (ni->ni_queue == NULL)
254 ni->ni_handler(NULL);
255 else
256 netisr_processqueue(ni);
252 if ((ni->ni_flags & NETISR_MPSAFE) == 0) {
253 mtx_lock(&Giant);
254 if (ni->ni_queue == NULL)
255 ni->ni_handler(NULL);
256 else
257 netisr_processqueue(ni);
258 mtx_unlock(&Giant);
259 } else {
260 if (ni->ni_queue == NULL)
261 ni->ni_handler(NULL);
262 else
263 netisr_processqueue(ni);
264 }
257 }
258 } while (polling);
265 }
266 } while (polling);
259 mtx_unlock(&netisr_mtx);
260}
261
262static void
263start_netisr(void *dummy)
264{
265
267}
268
269static void
270start_netisr(void *dummy)
271{
272
266 mtx_init(&netisr_mtx, "netisr lock", NULL, MTX_DEF);
267 if (swi_add(NULL, "net", swi_net, NULL, SWI_NET, 0, &net_ih))
273 if (swi_add(NULL, "net", swi_net, NULL, SWI_NET, INTR_MPSAFE, &net_ih))
268 panic("start_netisr");
269}
270SYSINIT(start_netisr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_netisr, NULL)
274 panic("start_netisr");
275}
276SYSINIT(start_netisr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_netisr, NULL)