netisr.c (122152) | netisr.c (122320) |
---|---|
1/*- 2 * Copyright (c) 2001,2002,2003 Jonathan Lemon <jlemon@FreeBSD.org> 3 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 10 unchanged lines hidden (view full) --- 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * | 1/*- 2 * Copyright (c) 2001,2002,2003 Jonathan Lemon <jlemon@FreeBSD.org> 3 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 10 unchanged lines hidden (view full) --- 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * |
27 * $FreeBSD: head/sys/net/netisr.c 122152 2003-11-05 23:42:51Z sam $ | 27 * $FreeBSD: head/sys/net/netisr.c 122320 2003-11-08 22:28:40Z sam $ |
28 */ 29 30#include <sys/param.h> 31#include <sys/bus.h> 32#include <sys/rtprio.h> 33#include <sys/systm.h> 34#include <sys/interrupt.h> 35#include <sys/kernel.h> --- 27 unchanged lines hidden (view full) --- 63SYSCTL_INT(_debug, OID_AUTO, mpsafenet, CTLFLAG_RD, &debug_mpsafenet, 0, 64 "Enable/disable MPSAFE network support"); 65 66volatile unsigned int netisr; /* scheduling bits for network */ 67 68struct netisr { 69 netisr_t *ni_handler; 70 struct ifqueue *ni_queue; | 28 */ 29 30#include <sys/param.h> 31#include <sys/bus.h> 32#include <sys/rtprio.h> 33#include <sys/systm.h> 34#include <sys/interrupt.h> 35#include <sys/kernel.h> --- 27 unchanged lines hidden (view full) --- 63SYSCTL_INT(_debug, OID_AUTO, mpsafenet, CTLFLAG_RD, &debug_mpsafenet, 0, 64 "Enable/disable MPSAFE network support"); 65 66volatile unsigned int netisr; /* scheduling bits for network */ 67 68struct netisr { 69 netisr_t *ni_handler; 70 struct ifqueue *ni_queue; |
71 int ni_flags; |
|
71} netisrs[32]; 72 | 72} netisrs[32]; 73 |
73static struct mtx netisr_mtx; | |
74static void *net_ih; 75 76void 77legacy_setsoftnet(void) 78{ 79 swi_sched(net_ih, 0); 80} 81 82void | 74static void *net_ih; 75 76void 77legacy_setsoftnet(void) 78{ 79 swi_sched(net_ih, 0); 80} 81 82void |
83netisr_register(int num, netisr_t *handler, struct ifqueue *inq) | 83netisr_register(int num, netisr_t *handler, struct ifqueue *inq, int flags) |
84{ 85 86 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 87 ("bad isr %d", num)); 88 netisrs[num].ni_handler = handler; 89 netisrs[num].ni_queue = inq; | 84{ 85 86 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 87 ("bad isr %d", num)); 88 netisrs[num].ni_handler = handler; 89 netisrs[num].ni_queue = inq; |
90 if ((flags & NETISR_MPSAFE) && !debug_mpsafenet) 91 flags &= ~NETISR_MPSAFE; 92 netisrs[num].ni_flags = flags; |
|
90} 91 92void 93netisr_unregister(int num) 94{ 95 struct netisr *ni; | 93} 94 95void 96netisr_unregister(int num) 97{ 98 struct netisr *ni; |
96 int s; | |
97 98 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 99 ("bad isr %d", num)); 100 ni = &netisrs[num]; 101 ni->ni_handler = NULL; | 99 100 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 101 ("bad isr %d", num)); 102 ni = &netisrs[num]; 103 ni->ni_handler = NULL; |
102 if (ni->ni_queue != NULL) { 103 s = splimp(); | 104 if (ni->ni_queue != NULL) |
104 IF_DRAIN(ni->ni_queue); | 105 IF_DRAIN(ni->ni_queue); |
105 splx(s); 106 } | |
107} 108 109struct isrstat { 110 int isrs_count; /* dispatch count */ | 106} 107 108struct isrstat { 109 int isrs_count; /* dispatch count */ |
111 int isrs_directed; /* ...successfully dispatched */ | 110 int isrs_directed; /* ...directly dispatched */ |
112 int isrs_deferred; /* ...queued instead */ 113 int isrs_queued; /* intentionally queueued */ | 111 int isrs_deferred; /* ...queued instead */ 112 int isrs_queued; /* intentionally queueued */ |
113 int isrs_drop; /* dropped 'cuz no handler */ |
|
114 int isrs_swi_count; /* swi_net handlers called */ 115}; 116static struct isrstat isrstat; 117 118SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr counters"); 119 120static int netisr_enable = 0; 121SYSCTL_INT(_net_isr, OID_AUTO, enable, CTLFLAG_RW, 122 &netisr_enable, 0, "enable direct dispatch"); 123TUNABLE_INT("net.isr.enable", &netisr_enable); 124 125SYSCTL_INT(_net_isr, OID_AUTO, count, CTLFLAG_RD, 126 &isrstat.isrs_count, 0, ""); 127SYSCTL_INT(_net_isr, OID_AUTO, directed, CTLFLAG_RD, 128 &isrstat.isrs_directed, 0, ""); 129SYSCTL_INT(_net_isr, OID_AUTO, deferred, CTLFLAG_RD, 130 &isrstat.isrs_deferred, 0, ""); 131SYSCTL_INT(_net_isr, OID_AUTO, queued, CTLFLAG_RD, 132 &isrstat.isrs_queued, 0, ""); | 114 int isrs_swi_count; /* swi_net handlers called */ 115}; 116static struct isrstat isrstat; 117 118SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr counters"); 119 120static int netisr_enable = 0; 121SYSCTL_INT(_net_isr, OID_AUTO, enable, CTLFLAG_RW, 122 &netisr_enable, 0, "enable direct dispatch"); 123TUNABLE_INT("net.isr.enable", &netisr_enable); 124 125SYSCTL_INT(_net_isr, OID_AUTO, count, CTLFLAG_RD, 126 &isrstat.isrs_count, 0, ""); 127SYSCTL_INT(_net_isr, OID_AUTO, directed, CTLFLAG_RD, 128 &isrstat.isrs_directed, 0, ""); 129SYSCTL_INT(_net_isr, OID_AUTO, deferred, CTLFLAG_RD, 130 &isrstat.isrs_deferred, 0, ""); 131SYSCTL_INT(_net_isr, OID_AUTO, queued, CTLFLAG_RD, 132 &isrstat.isrs_queued, 0, ""); |
133SYSCTL_INT(_net_isr, OID_AUTO, drop, CTLFLAG_RD, 134 &isrstat.isrs_drop, 0, ""); |
|
133SYSCTL_INT(_net_isr, OID_AUTO, swi_count, CTLFLAG_RD, 134 &isrstat.isrs_swi_count, 0, ""); 135 136/* 137 * Process all packets currently present in a netisr queue. Used to 138 * drain an existing set of packets waiting for processing when we 139 * begin direct dispatch, to avoid processing packets out of order. 140 */ --- 7 unchanged lines hidden (view full) --- 148 if (m == NULL) 149 break; 150 ni->ni_handler(m); 151 } 152} 153 154/* 155 * Call the netisr directly instead of queueing the packet, if possible. | 135SYSCTL_INT(_net_isr, OID_AUTO, swi_count, CTLFLAG_RD, 136 &isrstat.isrs_swi_count, 0, ""); 137 138/* 139 * Process all packets currently present in a netisr queue. Used to 140 * drain an existing set of packets waiting for processing when we 141 * begin direct dispatch, to avoid processing packets out of order. 142 */ --- 7 unchanged lines hidden (view full) --- 150 if (m == NULL) 151 break; 152 ni->ni_handler(m); 153 } 154} 155 156/* 157 * Call the netisr directly instead of queueing the packet, if possible. |
156 * 157 * Ideally, the permissibility of calling the routine would be determined 158 * by checking if splnet() was asserted at the time the device interrupt 159 * occurred; if so, this indicates that someone is in the network stack. 160 * 161 * However, bus_setup_intr uses INTR_TYPE_NET, which sets splnet before 162 * calling the interrupt handler, so the previous mask is unavailable. 163 * Approximate this by checking intr_nesting_level instead; if any SWI 164 * handlers are running, the packet is queued instead. | |
165 */ 166void 167netisr_dispatch(int num, struct mbuf *m) 168{ 169 struct netisr *ni; 170 | 158 */ 159void 160netisr_dispatch(int num, struct mbuf *m) 161{ 162 struct netisr *ni; 163 |
171 isrstat.isrs_count++; | 164 isrstat.isrs_count++; /* XXX redundant */ |
172 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 173 ("bad isr %d", num)); 174 ni = &netisrs[num]; 175 if (ni->ni_queue == NULL) { | 165 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 166 ("bad isr %d", num)); 167 ni = &netisrs[num]; 168 if (ni->ni_queue == NULL) { |
169 isrstat.isrs_drop++; |
|
176 m_freem(m); 177 return; 178 } | 170 m_freem(m); 171 return; 172 } |
179 if (netisr_enable && mtx_trylock(&netisr_mtx)) { | 173 /* 174 * Do direct dispatch only for MPSAFE netisrs (and 175 * only when enabled). Note that when a netisr is 176 * marked MPSAFE we permit multiple concurrent instances 177 * to run. We guarantee only the order in which 178 * packets are processed for each "dispatch point" in 179 * the system (i.e. call to netisr_dispatch or 180 * netisr_queue). This insures ordering of packets 181 * from an interface but does not guarantee ordering 182 * between multiple places in the system (e.g. IP 183 * dispatched from interfaces vs. IP queued from IPSec). 184 */ 185 if (netisr_enable && (ni->ni_flags & NETISR_MPSAFE)) { |
180 isrstat.isrs_directed++; 181 /* | 186 isrstat.isrs_directed++; 187 /* |
182 * One slight problem here is that packets might bypass 183 * each other in the stack, if an earlier one happened 184 * to get stuck in the queue. 185 * 186 * we can either: 187 * a. drain the queue before handling this packet, 188 * b. fallback to queueing the packet, 189 * c. sweep the issue under the rug and ignore it. 190 * 191 * Currently, we do a). Previously, we did c). | 188 * NB: We used to drain the queue before handling 189 * the packet but now do not. Doing so here will 190 * not preserve ordering so instead we fallback to 191 * guaranteeing order only from dispatch points 192 * in the system (see above). |
192 */ | 193 */ |
193 netisr_processqueue(ni); | |
194 ni->ni_handler(m); | 194 ni->ni_handler(m); |
195 mtx_unlock(&netisr_mtx); | |
196 } else { 197 isrstat.isrs_deferred++; 198 if (IF_HANDOFF(ni->ni_queue, m, NULL)) 199 schednetisr(num); 200 } 201} 202 203/* --- 5 unchanged lines hidden (view full) --- 209netisr_queue(int num, struct mbuf *m) 210{ 211 struct netisr *ni; 212 213 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 214 ("bad isr %d", num)); 215 ni = &netisrs[num]; 216 if (ni->ni_queue == NULL) { | 195 } else { 196 isrstat.isrs_deferred++; 197 if (IF_HANDOFF(ni->ni_queue, m, NULL)) 198 schednetisr(num); 199 } 200} 201 202/* --- 5 unchanged lines hidden (view full) --- 208netisr_queue(int num, struct mbuf *m) 209{ 210 struct netisr *ni; 211 212 KASSERT(!(num < 0 || num >= (sizeof(netisrs)/sizeof(*netisrs))), 213 ("bad isr %d", num)); 214 ni = &netisrs[num]; 215 if (ni->ni_queue == NULL) { |
216 isrstat.isrs_drop++; |
|
217 m_freem(m); 218 return (1); 219 } 220 isrstat.isrs_queued++; 221 if (!IF_HANDOFF(ni->ni_queue, m, NULL)) 222 return (0); 223 schednetisr(num); 224 return (1); --- 6 unchanged lines hidden (view full) --- 231 u_int bits; 232 int i; 233#ifdef DEVICE_POLLING 234 const int polling = 1; 235#else 236 const int polling = 0; 237#endif 238 | 217 m_freem(m); 218 return (1); 219 } 220 isrstat.isrs_queued++; 221 if (!IF_HANDOFF(ni->ni_queue, m, NULL)) 222 return (0); 223 schednetisr(num); 224 return (1); --- 6 unchanged lines hidden (view full) --- 231 u_int bits; 232 int i; 233#ifdef DEVICE_POLLING 234 const int polling = 1; 235#else 236 const int polling = 0; 237#endif 238 |
239 mtx_lock(&netisr_mtx); | |
240 do { 241 bits = atomic_readandclear_int(&netisr); 242 if (bits == 0) 243 break; 244 while ((i = ffs(bits)) != 0) { 245 isrstat.isrs_swi_count++; 246 i--; 247 bits &= ~(1 << i); 248 ni = &netisrs[i]; 249 if (ni->ni_handler == NULL) { 250 printf("swi_net: unregistered isr %d.\n", i); 251 continue; 252 } | 239 do { 240 bits = atomic_readandclear_int(&netisr); 241 if (bits == 0) 242 break; 243 while ((i = ffs(bits)) != 0) { 244 isrstat.isrs_swi_count++; 245 i--; 246 bits &= ~(1 << i); 247 ni = &netisrs[i]; 248 if (ni->ni_handler == NULL) { 249 printf("swi_net: unregistered isr %d.\n", i); 250 continue; 251 } |
253 if (ni->ni_queue == NULL) 254 ni->ni_handler(NULL); 255 else 256 netisr_processqueue(ni); | 252 if ((ni->ni_flags & NETISR_MPSAFE) == 0) { 253 mtx_lock(&Giant); 254 if (ni->ni_queue == NULL) 255 ni->ni_handler(NULL); 256 else 257 netisr_processqueue(ni); 258 mtx_unlock(&Giant); 259 } else { 260 if (ni->ni_queue == NULL) 261 ni->ni_handler(NULL); 262 else 263 netisr_processqueue(ni); 264 } |
257 } 258 } while (polling); | 265 } 266 } while (polling); |
259 mtx_unlock(&netisr_mtx); | |
260} 261 262static void 263start_netisr(void *dummy) 264{ 265 | 267} 268 269static void 270start_netisr(void *dummy) 271{ 272 |
266 mtx_init(&netisr_mtx, "netisr lock", NULL, MTX_DEF); 267 if (swi_add(NULL, "net", swi_net, NULL, SWI_NET, 0, &net_ih)) | 273 if (swi_add(NULL, "net", swi_net, NULL, SWI_NET, INTR_MPSAFE, &net_ih)) |
268 panic("start_netisr"); 269} 270SYSINIT(start_netisr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_netisr, NULL) | 274 panic("start_netisr"); 275} 276SYSINIT(start_netisr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_netisr, NULL) |