Deleted Added
sdiff udiff text old ( 267284 ) new ( 270063 )
full compact
netmap.c (267284) netmap.c (270063)
1/*
2 * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 11 unchanged lines hidden (view full) ---

20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26
27/*
1/*
2 * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 11 unchanged lines hidden (view full) ---

20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26
27/*
28 * $FreeBSD: head/sys/dev/netmap/netmap.c 267284 2014-06-09 15:46:11Z luigi $
28 * $FreeBSD: head/sys/dev/netmap/netmap.c 270063 2014-08-16 15:00:01Z luigi $
29 *
30 * This module supports memory mapped access to network devices,
31 * see netmap(4).
32 *
33 * The module uses a large, memory pool allocated by the kernel
34 * and accessible as mmapped memory by multiple userspace threads/processes.
35 * The memory pool contains packet buffers and "netmap rings",
36 * i.e. user-accessible copies of the interface's queues.

--- 82 unchanged lines hidden (view full) ---

119a number of slot in the ring, then the lock is released,
120packets are copied from source to destination, and then
121the lock is acquired again and the receive ring is updated.
122(A similar thing is done on the tx ring for NIC and host stack
123ports attached to the switch)
124
125 */
126
29 *
30 * This module supports memory mapped access to network devices,
31 * see netmap(4).
32 *
33 * The module uses a large, memory pool allocated by the kernel
34 * and accessible as mmapped memory by multiple userspace threads/processes.
35 * The memory pool contains packet buffers and "netmap rings",
36 * i.e. user-accessible copies of the interface's queues.

--- 82 unchanged lines hidden (view full) ---

119a number of slot in the ring, then the lock is released,
120packets are copied from source to destination, and then
121the lock is acquired again and the receive ring is updated.
122(A similar thing is done on the tx ring for NIC and host stack
123ports attached to the switch)
124
125 */
126
127
128/* --- internals ----
129 *
130 * Roadmap to the code that implements the above.
131 *
132 * > 1. a process/thread issues one or more open() on /dev/netmap, to create
133 * > select()able file descriptor on which events are reported.
134 *
135 * Internally, we allocate a netmap_priv_d structure, that will be
136 * initialized on ioctl(NIOCREGIF).
137 *
138 * os-specific:
139 * FreeBSD: netmap_open (netmap_freebsd.c). The priv is
140 * per-thread.
141 * linux: linux_netmap_open (netmap_linux.c). The priv is
142 * per-open.
143 *
144 * > 2. on each descriptor, the process issues an ioctl() to identify
145 * > the interface that should report events to the file descriptor.
146 *
147 * Implemented by netmap_ioctl(), NIOCREGIF case, with nmr->nr_cmd==0.
148 * Most important things happen in netmap_get_na() and
149 * netmap_do_regif(), called from there. Additional details can be
150 * found in the comments above those functions.
151 *
152 * In all cases, this action creates/takes-a-reference-to a
153 * netmap_*_adapter describing the port, and allocates a netmap_if
154 * and all necessary netmap rings, filling them with netmap buffers.
155 *
156 * In this phase, the sync callbacks for each ring are set (these are used
157 * in steps 5 and 6 below). The callbacks depend on the type of adapter.
158 * The adapter creation/initialization code puts them in the
159 * netmap_adapter (fields na->nm_txsync and na->nm_rxsync). Then, they
160 * are copied from there to the netmap_kring's during netmap_do_regif(), by
161 * the nm_krings_create() callback. All the nm_krings_create callbacks
162 * actually call netmap_krings_create() to perform this and the other
163 * common stuff. netmap_krings_create() also takes care of the host rings,
164 * if needed, by setting their sync callbacks appropriately.
165 *
166 * Additional actions depend on the kind of netmap_adapter that has been
167 * registered:
168 *
169 * - netmap_hw_adapter: [netmap.c]
170 * This is a system netdev/ifp with native netmap support.
171 * The ifp is detached from the host stack by redirecting:
172 * - transmissions (from the network stack) to netmap_transmit()
173 * - receive notifications to the nm_notify() callback for
174 * this adapter. The callback is normally netmap_notify(), unless
175 * the ifp is attached to a bridge using bwrap, in which case it
176 * is netmap_bwrap_intr_notify().
177 *
178 * - netmap_generic_adapter: [netmap_generic.c]
179 * A system netdev/ifp without native netmap support.
180 *
181 * (the decision about native/non native support is taken in
182 * netmap_get_hw_na(), called by netmap_get_na())
183 *
184 * - netmap_vp_adapter [netmap_vale.c]
185 * Returned by netmap_get_bdg_na().
186 * This is a persistent or ephemeral VALE port. Ephemeral ports
187 * are created on the fly if they don't already exist, and are
188 * always attached to a bridge.
189 * Persistent VALE ports must must be created seperately, and i
190 * then attached like normal NICs. The NIOCREGIF we are examining
191 * will find them only if they had previosly been created and
192 * attached (see VALE_CTL below).
193 *
194 * - netmap_pipe_adapter [netmap_pipe.c]
195 * Returned by netmap_get_pipe_na().
196 * Both pipe ends are created, if they didn't already exist.
197 *
198 * - netmap_monitor_adapter [netmap_monitor.c]
199 * Returned by netmap_get_monitor_na().
200 * If successful, the nm_sync callbacks of the monitored adapter
201 * will be intercepted by the returned monitor.
202 *
203 * - netmap_bwrap_adapter [netmap_vale.c]
204 * Cannot be obtained in this way, see VALE_CTL below
205 *
206 *
207 * os-specific:
208 * linux: we first go through linux_netmap_ioctl() to
209 * adapt the FreeBSD interface to the linux one.
210 *
211 *
212 * > 3. on each descriptor, the process issues an mmap() request to
213 * > map the shared memory region within the process' address space.
214 * > The list of interesting queues is indicated by a location in
215 * > the shared memory region.
216 *
217 * os-specific:
218 * FreeBSD: netmap_mmap_single (netmap_freebsd.c).
219 * linux: linux_netmap_mmap (netmap_linux.c).
220 *
221 * > 4. using the functions in the netmap(4) userspace API, a process
222 * > can look up the occupation state of a queue, access memory buffers,
223 * > and retrieve received packets or enqueue packets to transmit.
224 *
225 * these actions do not involve the kernel.
226 *
227 * > 5. using some ioctl()s the process can synchronize the userspace view
228 * > of the queue with the actual status in the kernel. This includes both
229 * > receiving the notification of new packets, and transmitting new
230 * > packets on the output interface.
231 *
232 * These are implemented in netmap_ioctl(), NIOCTXSYNC and NIOCRXSYNC
233 * cases. They invoke the nm_sync callbacks on the netmap_kring
234 * structures, as initialized in step 2 and maybe later modified
235 * by a monitor. Monitors, however, will always call the original
236 * callback before doing anything else.
237 *
238 *
239 * > 6. select() or poll() can be used to wait for events on individual
240 * > transmit or receive queues (or all queues for a given interface).
241 *
242 * Implemented in netmap_poll(). This will call the same nm_sync()
243 * callbacks as in step 5 above.
244 *
245 * os-specific:
246 * linux: we first go through linux_netmap_poll() to adapt
247 * the FreeBSD interface to the linux one.
248 *
249 *
250 * ---- VALE_CTL -----
251 *
252 * VALE switches are controlled by issuing a NIOCREGIF with a non-null
253 * nr_cmd in the nmreq structure. These subcommands are handled by
254 * netmap_bdg_ctl() in netmap_vale.c. Persistent VALE ports are created
255 * and destroyed by issuing the NETMAP_BDG_NEWIF and NETMAP_BDG_DELIF
256 * subcommands, respectively.
257 *
258 * Any network interface known to the system (including a persistent VALE
259 * port) can be attached to a VALE switch by issuing the
260 * NETMAP_BDG_ATTACH subcommand. After the attachment, persistent VALE ports
261 * look exactly like ephemeral VALE ports (as created in step 2 above). The
262 * attachment of other interfaces, instead, requires the creation of a
263 * netmap_bwrap_adapter. Moreover, the attached interface must be put in
264 * netmap mode. This may require the creation of a netmap_generic_adapter if
265 * we have no native support for the interface, or if generic adapters have
266 * been forced by sysctl.
267 *
268 * Both persistent VALE ports and bwraps are handled by netmap_get_bdg_na(),
269 * called by nm_bdg_ctl_attach(), and discriminated by the nm_bdg_attach()
270 * callback. In the case of the bwrap, the callback creates the
271 * netmap_bwrap_adapter. The initialization of the bwrap is then
272 * completed by calling netmap_do_regif() on it, in the nm_bdg_ctl()
273 * callback (netmap_bwrap_bdg_ctl in netmap_vale.c).
274 * A generic adapter for the wrapped ifp will be created if needed, when
275 * netmap_get_bdg_na() calls netmap_get_hw_na().
276 *
277 *
278 * ---- DATAPATHS -----
279 *
280 * -= SYSTEM DEVICE WITH NATIVE SUPPORT =-
281 *
282 * na == NA(ifp) == netmap_hw_adapter created in DEVICE_netmap_attach()
283 *
284 * - tx from netmap userspace:
285 * concurrently:
286 * 1) ioctl(NIOCTXSYNC)/netmap_poll() in process context
287 * kring->nm_sync() == DEVICE_netmap_txsync()
288 * 2) device interrupt handler
289 * na->nm_notify() == netmap_notify()
290 * - rx from netmap userspace:
291 * concurrently:
292 * 1) ioctl(NIOCRXSYNC)/netmap_poll() in process context
293 * kring->nm_sync() == DEVICE_netmap_rxsync()
294 * 2) device interrupt handler
295 * na->nm_notify() == netmap_notify()
296 * - tx from host stack
297 * concurrently:
298 * 1) host stack
299 * netmap_transmit()
300 * na->nm_notify == netmap_notify()
301 * 2) ioctl(NIOCRXSYNC)/netmap_poll() in process context
302 * kring->nm_sync() == netmap_rxsync_from_host_compat
303 * netmap_rxsync_from_host(na, NULL, NULL)
304 * - tx to host stack
305 * ioctl(NIOCTXSYNC)/netmap_poll() in process context
306 * kring->nm_sync() == netmap_txsync_to_host_compat
307 * netmap_txsync_to_host(na)
308 * NM_SEND_UP()
309 * FreeBSD: na->if_input() == ?? XXX
310 * linux: netif_rx() with NM_MAGIC_PRIORITY_RX
311 *
312 *
313 *
314 * -= SYSTEM DEVICE WITH GENERIC SUPPORT =-
315 *
316 *
317 *
318 * -= VALE PORT =-
319 *
320 *
321 *
322 * -= NETMAP PIPE =-
323 *
324 *
325 *
326 * -= SYSTEM DEVICE WITH NATIVE SUPPORT, CONNECTED TO VALE, NO HOST RINGS =-
327 *
328 *
329 *
330 * -= SYSTEM DEVICE WITH NATIVE SUPPORT, CONNECTED TO VALE, WITH HOST RINGS =-
331 *
332 *
333 *
334 * -= SYSTEM DEVICE WITH GENERIC SUPPORT, CONNECTED TO VALE, NO HOST RINGS =-
335 *
336 *
337 *
338 * -= SYSTEM DEVICE WITH GENERIC SUPPORT, CONNECTED TO VALE, WITH HOST RINGS =-
339 *
340 *
341 *
342 */
343
127/*
128 * OS-specific code that is used only within this file.
129 * Other OS-specific code that must be accessed by drivers
130 * is present in netmap_kern.h
131 */
132
133#if defined(__FreeBSD__)
134#include <sys/cdefs.h> /* prerequisite */

--- 78 unchanged lines hidden (view full) ---

213SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, "");
214int netmap_no_pendintr = 1;
215SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr,
216 CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets.");
217int netmap_txsync_retry = 2;
218SYSCTL_INT(_dev_netmap, OID_AUTO, txsync_retry, CTLFLAG_RW,
219 &netmap_txsync_retry, 0 , "Number of txsync loops in bridge's flush.");
220
344/*
345 * OS-specific code that is used only within this file.
346 * Other OS-specific code that must be accessed by drivers
347 * is present in netmap_kern.h
348 */
349
350#if defined(__FreeBSD__)
351#include <sys/cdefs.h> /* prerequisite */

--- 78 unchanged lines hidden (view full) ---

430SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, "");
431int netmap_no_pendintr = 1;
432SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr,
433 CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets.");
434int netmap_txsync_retry = 2;
435SYSCTL_INT(_dev_netmap, OID_AUTO, txsync_retry, CTLFLAG_RW,
436 &netmap_txsync_retry, 0 , "Number of txsync loops in bridge's flush.");
437
438int netmap_adaptive_io = 0;
439SYSCTL_INT(_dev_netmap, OID_AUTO, adaptive_io, CTLFLAG_RW,
440 &netmap_adaptive_io, 0 , "Adaptive I/O on paravirt");
441
221int netmap_flags = 0; /* debug flags */
222int netmap_fwd = 0; /* force transparent mode */
223int netmap_mmap_unreg = 0; /* allow mmap of unregistered fds */
224
225/*
226 * netmap_admode selects the netmap mode to use.
227 * Invalid values are reset to NETMAP_ADMODE_BEST
228 */

--- 25 unchanged lines hidden (view full) ---

254 tsleep(kr, 0, "NM_KR_GET", 4);
255}
256
257
258/*
259 * mark the ring as stopped, and run through the locks
260 * to make sure other users get to see it.
261 */
442int netmap_flags = 0; /* debug flags */
443int netmap_fwd = 0; /* force transparent mode */
444int netmap_mmap_unreg = 0; /* allow mmap of unregistered fds */
445
446/*
447 * netmap_admode selects the netmap mode to use.
448 * Invalid values are reset to NETMAP_ADMODE_BEST
449 */

--- 25 unchanged lines hidden (view full) ---

475 tsleep(kr, 0, "NM_KR_GET", 4);
476}
477
478
479/*
480 * mark the ring as stopped, and run through the locks
481 * to make sure other users get to see it.
482 */
262void
483static void
263netmap_disable_ring(struct netmap_kring *kr)
264{
265 kr->nkr_stopped = 1;
266 nm_kr_get(kr);
267 mtx_lock(&kr->q_lock);
268 mtx_unlock(&kr->q_lock);
269 nm_kr_put(kr);
270}
271
484netmap_disable_ring(struct netmap_kring *kr)
485{
486 kr->nkr_stopped = 1;
487 nm_kr_get(kr);
488 mtx_lock(&kr->q_lock);
489 mtx_unlock(&kr->q_lock);
490 nm_kr_put(kr);
491}
492
493/* stop or enable a single tx ring */
494void
495netmap_set_txring(struct netmap_adapter *na, u_int ring_id, int stopped)
496{
497 if (stopped)
498 netmap_disable_ring(na->tx_rings + ring_id);
499 else
500 na->tx_rings[ring_id].nkr_stopped = 0;
501 /* nofify that the stopped state has changed. This is currently
502 *only used by bwrap to propagate the state to its own krings.
503 * (see netmap_bwrap_intr_notify).
504 */
505 na->nm_notify(na, ring_id, NR_TX, NAF_DISABLE_NOTIFY);
506}
272
507
508/* stop or enable a single rx ring */
509void
510netmap_set_rxring(struct netmap_adapter *na, u_int ring_id, int stopped)
511{
512 if (stopped)
513 netmap_disable_ring(na->rx_rings + ring_id);
514 else
515 na->rx_rings[ring_id].nkr_stopped = 0;
516 /* nofify that the stopped state has changed. This is currently
517 *only used by bwrap to propagate the state to its own krings.
518 * (see netmap_bwrap_intr_notify).
519 */
520 na->nm_notify(na, ring_id, NR_RX, NAF_DISABLE_NOTIFY);
521}
522
523
273/* stop or enable all the rings of na */
524/* stop or enable all the rings of na */
274static void
275netmap_set_all_rings(struct ifnet *ifp, int stopped)
525void
526netmap_set_all_rings(struct netmap_adapter *na, int stopped)
276{
527{
277 struct netmap_adapter *na;
278 int i;
279 u_int ntx, nrx;
280
528 int i;
529 u_int ntx, nrx;
530
281 if (!(ifp->if_capenable & IFCAP_NETMAP))
531 if (!nm_netmap_on(na))
282 return;
283
532 return;
533
284 na = NA(ifp);
285
286 ntx = netmap_real_tx_rings(na);
287 nrx = netmap_real_rx_rings(na);
288
289 for (i = 0; i < ntx; i++) {
534 ntx = netmap_real_tx_rings(na);
535 nrx = netmap_real_rx_rings(na);
536
537 for (i = 0; i < ntx; i++) {
290 if (stopped)
291 netmap_disable_ring(na->tx_rings + i);
292 else
293 na->tx_rings[i].nkr_stopped = 0;
294 na->nm_notify(na, i, NR_TX, NAF_DISABLE_NOTIFY);
538 netmap_set_txring(na, i, stopped);
295 }
296
297 for (i = 0; i < nrx; i++) {
539 }
540
541 for (i = 0; i < nrx; i++) {
298 if (stopped)
299 netmap_disable_ring(na->rx_rings + i);
300 else
301 na->rx_rings[i].nkr_stopped = 0;
302 na->nm_notify(na, i, NR_RX, NAF_DISABLE_NOTIFY);
542 netmap_set_rxring(na, i, stopped);
303 }
304}
305
543 }
544}
545
306
307/*
308 * Convenience function used in drivers. Waits for current txsync()s/rxsync()s
309 * to finish and prevents any new one from starting. Call this before turning
310 * netmap mode off, or before removing the harware rings (e.g., on module
311 * onload). As a rule of thumb for linux drivers, this should be placed near
312 * each napi_disable().
313 */
314void
315netmap_disable_all_rings(struct ifnet *ifp)
316{
546/*
547 * Convenience function used in drivers. Waits for current txsync()s/rxsync()s
548 * to finish and prevents any new one from starting. Call this before turning
549 * netmap mode off, or before removing the harware rings (e.g., on module
550 * onload). As a rule of thumb for linux drivers, this should be placed near
551 * each napi_disable().
552 */
553void
554netmap_disable_all_rings(struct ifnet *ifp)
555{
317 netmap_set_all_rings(ifp, 1 /* stopped */);
556 netmap_set_all_rings(NA(ifp), 1 /* stopped */);
318}
319
557}
558
320
321/*
322 * Convenience function used in drivers. Re-enables rxsync and txsync on the
323 * adapter's rings In linux drivers, this should be placed near each
324 * napi_enable().
325 */
326void
327netmap_enable_all_rings(struct ifnet *ifp)
328{
559/*
560 * Convenience function used in drivers. Re-enables rxsync and txsync on the
561 * adapter's rings In linux drivers, this should be placed near each
562 * napi_enable().
563 */
564void
565netmap_enable_all_rings(struct ifnet *ifp)
566{
329 netmap_set_all_rings(ifp, 0 /* enabled */);
567 netmap_set_all_rings(NA(ifp), 0 /* enabled */);
330}
331
332
333/*
334 * generic bound_checking function
335 */
336u_int
337nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg)

--- 67 unchanged lines hidden (view full) ---

405/*
406 * Fetch configuration from the device, to cope with dynamic
407 * reconfigurations after loading the module.
408 */
409/* call with NMG_LOCK held */
410int
411netmap_update_config(struct netmap_adapter *na)
412{
568}
569
570
571/*
572 * generic bound_checking function
573 */
574u_int
575nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg)

--- 67 unchanged lines hidden (view full) ---

643/*
644 * Fetch configuration from the device, to cope with dynamic
645 * reconfigurations after loading the module.
646 */
647/* call with NMG_LOCK held */
648int
649netmap_update_config(struct netmap_adapter *na)
650{
413 struct ifnet *ifp = na->ifp;
414 u_int txr, txd, rxr, rxd;
415
416 txr = txd = rxr = rxd = 0;
417 if (na->nm_config) {
418 na->nm_config(na, &txr, &txd, &rxr, &rxd);
419 } else {
420 /* take whatever we had at init time */
421 txr = na->num_tx_rings;
422 txd = na->num_tx_desc;
423 rxr = na->num_rx_rings;
424 rxd = na->num_rx_desc;
425 }
426
427 if (na->num_tx_rings == txr && na->num_tx_desc == txd &&
428 na->num_rx_rings == rxr && na->num_rx_desc == rxd)
429 return 0; /* nothing changed */
430 if (netmap_verbose || na->active_fds > 0) {
431 D("stored config %s: txring %d x %d, rxring %d x %d",
651 u_int txr, txd, rxr, rxd;
652
653 txr = txd = rxr = rxd = 0;
654 if (na->nm_config) {
655 na->nm_config(na, &txr, &txd, &rxr, &rxd);
656 } else {
657 /* take whatever we had at init time */
658 txr = na->num_tx_rings;
659 txd = na->num_tx_desc;
660 rxr = na->num_rx_rings;
661 rxd = na->num_rx_desc;
662 }
663
664 if (na->num_tx_rings == txr && na->num_tx_desc == txd &&
665 na->num_rx_rings == rxr && na->num_rx_desc == rxd)
666 return 0; /* nothing changed */
667 if (netmap_verbose || na->active_fds > 0) {
668 D("stored config %s: txring %d x %d, rxring %d x %d",
432 NM_IFPNAME(ifp),
669 na->name,
433 na->num_tx_rings, na->num_tx_desc,
434 na->num_rx_rings, na->num_rx_desc);
435 D("new config %s: txring %d x %d, rxring %d x %d",
670 na->num_tx_rings, na->num_tx_desc,
671 na->num_rx_rings, na->num_rx_desc);
672 D("new config %s: txring %d x %d, rxring %d x %d",
436 NM_IFPNAME(ifp), txr, txd, rxr, rxd);
673 na->name, txr, txd, rxr, rxd);
437 }
438 if (na->active_fds == 0) {
439 D("configuration changed (but fine)");
440 na->num_tx_rings = txr;
441 na->num_tx_desc = txd;
442 na->num_rx_rings = rxr;
443 na->num_rx_desc = rxd;
444 return 0;
445 }
446 D("configuration changed while active, this is bad...");
447 return 1;
448}
449
674 }
675 if (na->active_fds == 0) {
676 D("configuration changed (but fine)");
677 na->num_tx_rings = txr;
678 na->num_tx_desc = txd;
679 na->num_rx_rings = rxr;
680 na->num_rx_desc = rxd;
681 return 0;
682 }
683 D("configuration changed while active, this is bad...");
684 return 1;
685}
686
450static int
451netmap_txsync_compat(struct netmap_kring *kring, int flags)
452{
453 struct netmap_adapter *na = kring->na;
454 return na->nm_txsync(na, kring->ring_id, flags);
455}
456
457static int
458netmap_rxsync_compat(struct netmap_kring *kring, int flags)
459{
460 struct netmap_adapter *na = kring->na;
461 return na->nm_rxsync(na, kring->ring_id, flags);
462}
463
464/* kring->nm_sync callback for the host tx ring */
465static int
466netmap_txsync_to_host_compat(struct netmap_kring *kring, int flags)
467{
468 (void)flags; /* unused */
469 netmap_txsync_to_host(kring->na);
470 return 0;
471}

--- 61 unchanged lines hidden (view full) ---

533 ndesc = na->num_tx_desc;
534 for (i = 0; i < ntx; i++) { /* Transmit rings */
535 kring = &na->tx_rings[i];
536 bzero(kring, sizeof(*kring));
537 kring->na = na;
538 kring->ring_id = i;
539 kring->nkr_num_slots = ndesc;
540 if (i < na->num_tx_rings) {
687/* kring->nm_sync callback for the host tx ring */
688static int
689netmap_txsync_to_host_compat(struct netmap_kring *kring, int flags)
690{
691 (void)flags; /* unused */
692 netmap_txsync_to_host(kring->na);
693 return 0;
694}

--- 61 unchanged lines hidden (view full) ---

756 ndesc = na->num_tx_desc;
757 for (i = 0; i < ntx; i++) { /* Transmit rings */
758 kring = &na->tx_rings[i];
759 bzero(kring, sizeof(*kring));
760 kring->na = na;
761 kring->ring_id = i;
762 kring->nkr_num_slots = ndesc;
763 if (i < na->num_tx_rings) {
541 kring->nm_sync = netmap_txsync_compat; // XXX
764 kring->nm_sync = na->nm_txsync;
542 } else if (i == na->num_tx_rings) {
543 kring->nm_sync = netmap_txsync_to_host_compat;
544 }
545 /*
546 * IMPORTANT: Always keep one slot empty.
547 */
548 kring->rhead = kring->rcur = kring->nr_hwcur = 0;
549 kring->rtail = kring->nr_hwtail = ndesc - 1;
765 } else if (i == na->num_tx_rings) {
766 kring->nm_sync = netmap_txsync_to_host_compat;
767 }
768 /*
769 * IMPORTANT: Always keep one slot empty.
770 */
771 kring->rhead = kring->rcur = kring->nr_hwcur = 0;
772 kring->rtail = kring->nr_hwtail = ndesc - 1;
550 snprintf(kring->name, sizeof(kring->name) - 1, "%s TX%d", NM_IFPNAME(na->ifp), i);
773 snprintf(kring->name, sizeof(kring->name) - 1, "%s TX%d", na->name, i);
551 ND("ktx %s h %d c %d t %d",
552 kring->name, kring->rhead, kring->rcur, kring->rtail);
553 mtx_init(&kring->q_lock, "nm_txq_lock", NULL, MTX_DEF);
554 init_waitqueue_head(&kring->si);
555 }
556
557 ndesc = na->num_rx_desc;
558 for (i = 0; i < nrx; i++) { /* Receive rings */
559 kring = &na->rx_rings[i];
560 bzero(kring, sizeof(*kring));
561 kring->na = na;
562 kring->ring_id = i;
563 kring->nkr_num_slots = ndesc;
564 if (i < na->num_rx_rings) {
774 ND("ktx %s h %d c %d t %d",
775 kring->name, kring->rhead, kring->rcur, kring->rtail);
776 mtx_init(&kring->q_lock, "nm_txq_lock", NULL, MTX_DEF);
777 init_waitqueue_head(&kring->si);
778 }
779
780 ndesc = na->num_rx_desc;
781 for (i = 0; i < nrx; i++) { /* Receive rings */
782 kring = &na->rx_rings[i];
783 bzero(kring, sizeof(*kring));
784 kring->na = na;
785 kring->ring_id = i;
786 kring->nkr_num_slots = ndesc;
787 if (i < na->num_rx_rings) {
565 kring->nm_sync = netmap_rxsync_compat; // XXX
788 kring->nm_sync = na->nm_rxsync;
566 } else if (i == na->num_rx_rings) {
567 kring->nm_sync = netmap_rxsync_from_host_compat;
568 }
569 kring->rhead = kring->rcur = kring->nr_hwcur = 0;
570 kring->rtail = kring->nr_hwtail = 0;
789 } else if (i == na->num_rx_rings) {
790 kring->nm_sync = netmap_rxsync_from_host_compat;
791 }
792 kring->rhead = kring->rcur = kring->nr_hwcur = 0;
793 kring->rtail = kring->nr_hwtail = 0;
571 snprintf(kring->name, sizeof(kring->name) - 1, "%s RX%d", NM_IFPNAME(na->ifp), i);
794 snprintf(kring->name, sizeof(kring->name) - 1, "%s RX%d", na->name, i);
572 ND("krx %s h %d c %d t %d",
573 kring->name, kring->rhead, kring->rcur, kring->rtail);
574 mtx_init(&kring->q_lock, "nm_rxq_lock", NULL, MTX_DEF);
575 init_waitqueue_head(&kring->si);
576 }
577 init_waitqueue_head(&na->tx_si);
578 init_waitqueue_head(&na->rx_si);
579

--- 39 unchanged lines hidden (view full) ---

619
620/* create a new netmap_if for a newly registered fd.
621 * If this is the first registration of the adapter,
622 * also create the netmap rings and their in-kernel view,
623 * the netmap krings.
624 */
625/* call with NMG_LOCK held */
626static struct netmap_if*
795 ND("krx %s h %d c %d t %d",
796 kring->name, kring->rhead, kring->rcur, kring->rtail);
797 mtx_init(&kring->q_lock, "nm_rxq_lock", NULL, MTX_DEF);
798 init_waitqueue_head(&kring->si);
799 }
800 init_waitqueue_head(&na->tx_si);
801 init_waitqueue_head(&na->rx_si);
802

--- 39 unchanged lines hidden (view full) ---

842
843/* create a new netmap_if for a newly registered fd.
844 * If this is the first registration of the adapter,
845 * also create the netmap rings and their in-kernel view,
846 * the netmap krings.
847 */
848/* call with NMG_LOCK held */
849static struct netmap_if*
627netmap_if_new(const char *ifname, struct netmap_adapter *na)
850netmap_if_new(struct netmap_adapter *na)
628{
629 struct netmap_if *nifp;
630
631 if (netmap_update_config(na)) {
632 /* configuration mismatch, report and fail */
633 return NULL;
634 }
635
636 if (na->active_fds) /* already registered */
637 goto final;
638
639 /* create and init the krings arrays.
640 * Depending on the adapter, this may also create
641 * the netmap rings themselves
642 */
643 if (na->nm_krings_create(na))
851{
852 struct netmap_if *nifp;
853
854 if (netmap_update_config(na)) {
855 /* configuration mismatch, report and fail */
856 return NULL;
857 }
858
859 if (na->active_fds) /* already registered */
860 goto final;
861
862 /* create and init the krings arrays.
863 * Depending on the adapter, this may also create
864 * the netmap rings themselves
865 */
866 if (na->nm_krings_create(na))
644 goto cleanup;
867 return NULL;
645
646 /* create all missing netmap rings */
647 if (netmap_mem_rings_create(na))
648 goto cleanup;
649
650final:
651
652 /* in all cases, create a new netmap if */
868
869 /* create all missing netmap rings */
870 if (netmap_mem_rings_create(na))
871 goto cleanup;
872
873final:
874
875 /* in all cases, create a new netmap if */
653 nifp = netmap_mem_if_new(ifname, na);
876 nifp = netmap_mem_if_new(na);
654 if (nifp == NULL)
655 goto cleanup;
656
657 return (nifp);
658
659cleanup:
660
661 if (na->active_fds == 0) {

--- 22 unchanged lines hidden (view full) ---

684 * we use the global allocator when no interface has been
685 * registered
686 */
687 nmd = &nm_mem;
688 } else {
689 nmd = p->np_na->nm_mem;
690 }
691 if (p->np_mref == NULL) {
877 if (nifp == NULL)
878 goto cleanup;
879
880 return (nifp);
881
882cleanup:
883
884 if (na->active_fds == 0) {

--- 22 unchanged lines hidden (view full) ---

907 * we use the global allocator when no interface has been
908 * registered
909 */
910 nmd = &nm_mem;
911 } else {
912 nmd = p->np_na->nm_mem;
913 }
914 if (p->np_mref == NULL) {
692 error = netmap_mem_finalize(nmd);
915 error = netmap_mem_finalize(nmd, p->np_na);
693 if (!error)
694 p->np_mref = nmd;
695 } else if (p->np_mref != nmd) {
696 /* a virtual port has been registered, but previous
697 * syscalls already used the global allocator.
698 * We cannot continue
699 */
700 error = ENODEV;

--- 22 unchanged lines hidden (view full) ---

723}
724
725
726/* call with NMG_LOCK held */
727static void
728netmap_drop_memory_locked(struct netmap_priv_d* p)
729{
730 if (p->np_mref) {
916 if (!error)
917 p->np_mref = nmd;
918 } else if (p->np_mref != nmd) {
919 /* a virtual port has been registered, but previous
920 * syscalls already used the global allocator.
921 * We cannot continue
922 */
923 error = ENODEV;

--- 22 unchanged lines hidden (view full) ---

946}
947
948
949/* call with NMG_LOCK held */
950static void
951netmap_drop_memory_locked(struct netmap_priv_d* p)
952{
953 if (p->np_mref) {
731 netmap_mem_deref(p->np_mref);
954 netmap_mem_deref(p->np_mref, p->np_na);
732 p->np_mref = NULL;
733 }
734}
735
736
737/*
955 p->np_mref = NULL;
956 }
957}
958
959
960/*
738 * File descriptor's private data destructor.
739 *
740 * Call nm_register(ifp,0) to stop netmap mode on the interface and
961 * Call nm_register(ifp,0) to stop netmap mode on the interface and
741 * revert to normal operation. We expect that np_na->ifp has not gone.
962 * revert to normal operation.
742 * The second argument is the nifp to work on. In some cases it is
743 * not attached yet to the netmap_priv_d so we need to pass it as
744 * a separate argument.
745 */
746/* call with NMG_LOCK held */
747static void
748netmap_do_unregif(struct netmap_priv_d *priv, struct netmap_if *nifp)
749{
750 struct netmap_adapter *na = priv->np_na;
963 * The second argument is the nifp to work on. In some cases it is
964 * not attached yet to the netmap_priv_d so we need to pass it as
965 * a separate argument.
966 */
967/* call with NMG_LOCK held */
968static void
969netmap_do_unregif(struct netmap_priv_d *priv, struct netmap_if *nifp)
970{
971 struct netmap_adapter *na = priv->np_na;
751 struct ifnet *ifp = na->ifp;
752
753 NMG_LOCK_ASSERT();
754 na->active_fds--;
755 if (na->active_fds <= 0) { /* last instance */
756
757 if (netmap_verbose)
972
973 NMG_LOCK_ASSERT();
974 na->active_fds--;
975 if (na->active_fds <= 0) { /* last instance */
976
977 if (netmap_verbose)
758 D("deleting last instance for %s", NM_IFPNAME(ifp));
978 D("deleting last instance for %s", na->name);
759 /*
760 * (TO CHECK) This function is only called
761 * when the last reference to this file descriptor goes
762 * away. This means we cannot have any pending poll()
763 * or interrupt routine operating on the structure.
764 * XXX The file may be closed in a thread while
765 * another thread is using it.
766 * Linux keeps the file opened until the last reference
767 * by any outstanding ioctl/poll or mmap is gone.
768 * FreeBSD does not track mmap()s (but we do) and
769 * wakes up any sleeping poll(). Need to check what
770 * happens if the close() occurs while a concurrent
771 * syscall is running.
772 */
979 /*
980 * (TO CHECK) This function is only called
981 * when the last reference to this file descriptor goes
982 * away. This means we cannot have any pending poll()
983 * or interrupt routine operating on the structure.
984 * XXX The file may be closed in a thread while
985 * another thread is using it.
986 * Linux keeps the file opened until the last reference
987 * by any outstanding ioctl/poll or mmap is gone.
988 * FreeBSD does not track mmap()s (but we do) and
989 * wakes up any sleeping poll(). Need to check what
990 * happens if the close() occurs while a concurrent
991 * syscall is running.
992 */
773 if (ifp)
774 na->nm_register(na, 0); /* off, clear flags */
993 na->nm_register(na, 0); /* off, clear flags */
775 /* Wake up any sleeping threads. netmap_poll will
776 * then return POLLERR
777 * XXX The wake up now must happen during *_down(), when
778 * we order all activities to stop. -gl
779 */
780 /* XXX kqueue(9) needed; these will mirror knlist_init. */
781 /* knlist_destroy(&na->tx_si.si_note); */
782 /* knlist_destroy(&na->rx_si.si_note); */

--- 134 unchanged lines hidden (view full) ---

917 struct netmap_adapter *na = kring->na;
918
919 for (n = kring->nr_hwcur; n != head; n = nm_next(n, lim)) {
920 struct mbuf *m;
921 struct netmap_slot *slot = &kring->ring->slot[n];
922
923 if ((slot->flags & NS_FORWARD) == 0 && !force)
924 continue;
994 /* Wake up any sleeping threads. netmap_poll will
995 * then return POLLERR
996 * XXX The wake up now must happen during *_down(), when
997 * we order all activities to stop. -gl
998 */
999 /* XXX kqueue(9) needed; these will mirror knlist_init. */
1000 /* knlist_destroy(&na->tx_si.si_note); */
1001 /* knlist_destroy(&na->rx_si.si_note); */

--- 134 unchanged lines hidden (view full) ---

1136 struct netmap_adapter *na = kring->na;
1137
1138 for (n = kring->nr_hwcur; n != head; n = nm_next(n, lim)) {
1139 struct mbuf *m;
1140 struct netmap_slot *slot = &kring->ring->slot[n];
1141
1142 if ((slot->flags & NS_FORWARD) == 0 && !force)
1143 continue;
925 if (slot->len < 14 || slot->len > NETMAP_BDG_BUF_SIZE(na->nm_mem)) {
1144 if (slot->len < 14 || slot->len > NETMAP_BUF_SIZE(na)) {
926 RD(5, "bad pkt at %d len %d", n, slot->len);
927 continue;
928 }
929 slot->flags &= ~NS_FORWARD; // XXX needed ?
930 /* XXX TODO: adapt to the case of a multisegment packet */
1145 RD(5, "bad pkt at %d len %d", n, slot->len);
1146 continue;
1147 }
1148 slot->flags &= ~NS_FORWARD; // XXX needed ?
1149 /* XXX TODO: adapt to the case of a multisegment packet */
931 m = m_devget(BDG_NMB(na, slot), slot->len, 0, na->ifp, NULL);
1150 m = m_devget(NMB(na, slot), slot->len, 0, na->ifp, NULL);
932
933 if (m == NULL)
934 break;
935 mbq_enqueue(q, m);
936 }
937}
938
939

--- 36 unchanged lines hidden (view full) ---

976
977 src->buf_idx = dst->buf_idx;
978 src->flags = NS_BUF_CHANGED;
979
980 dst->buf_idx = tmp.buf_idx;
981 dst->len = tmp.len;
982 dst->flags = NS_BUF_CHANGED;
983
1151
1152 if (m == NULL)
1153 break;
1154 mbq_enqueue(q, m);
1155 }
1156}
1157
1158

--- 36 unchanged lines hidden (view full) ---

1195
1196 src->buf_idx = dst->buf_idx;
1197 src->flags = NS_BUF_CHANGED;
1198
1199 dst->buf_idx = tmp.buf_idx;
1200 dst->len = tmp.len;
1201 dst->flags = NS_BUF_CHANGED;
1202
984 rdst->head = rdst->cur = nm_next(dst_cur, dst_lim);
1203 rdst->cur = nm_next(dst_cur, dst_lim);
985 }
986 /* if (sent) XXX txsync ? */
987 }
988 return sent;
989}
990
991
992/*

--- 30 unchanged lines hidden (view full) ---

1023}
1024
1025
1026/*
1027 * rxsync backend for packets coming from the host stack.
1028 * They have been put in kring->rx_queue by netmap_transmit().
1029 * We protect access to the kring using kring->rx_queue.lock
1030 *
1204 }
1205 /* if (sent) XXX txsync ? */
1206 }
1207 return sent;
1208}
1209
1210
1211/*

--- 30 unchanged lines hidden (view full) ---

1242}
1243
1244
1245/*
1246 * rxsync backend for packets coming from the host stack.
1247 * They have been put in kring->rx_queue by netmap_transmit().
1248 * We protect access to the kring using kring->rx_queue.lock
1249 *
1250 * This routine also does the selrecord if called from the poll handler
1251 * (we know because td != NULL).
1252 *
1253 * NOTE: on linux, selrecord() is defined as a macro and uses pwait
1254 * as an additional hidden argument.
1031 * returns the number of packets delivered to tx queues in
1032 * transparent mode, or a negative value if error
1033 */
1034int
1035netmap_rxsync_from_host(struct netmap_adapter *na, struct thread *td, void *pwait)
1036{
1037 struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];
1038 struct netmap_ring *ring = kring->ring;

--- 15 unchanged lines hidden (view full) ---

1054 uint32_t stop_i;
1055
1056 nm_i = kring->nr_hwtail;
1057 stop_i = nm_prev(nm_i, lim);
1058 while ( nm_i != stop_i && (m = mbq_dequeue(q)) != NULL ) {
1059 int len = MBUF_LEN(m);
1060 struct netmap_slot *slot = &ring->slot[nm_i];
1061
1255 * returns the number of packets delivered to tx queues in
1256 * transparent mode, or a negative value if error
1257 */
1258int
1259netmap_rxsync_from_host(struct netmap_adapter *na, struct thread *td, void *pwait)
1260{
1261 struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];
1262 struct netmap_ring *ring = kring->ring;

--- 15 unchanged lines hidden (view full) ---

1278 uint32_t stop_i;
1279
1280 nm_i = kring->nr_hwtail;
1281 stop_i = nm_prev(nm_i, lim);
1282 while ( nm_i != stop_i && (m = mbq_dequeue(q)) != NULL ) {
1283 int len = MBUF_LEN(m);
1284 struct netmap_slot *slot = &ring->slot[nm_i];
1285
1062 m_copydata(m, 0, len, BDG_NMB(na, slot));
1286 m_copydata(m, 0, len, NMB(na, slot));
1063 ND("nm %d len %d", nm_i, len);
1064 if (netmap_verbose)
1287 ND("nm %d len %d", nm_i, len);
1288 if (netmap_verbose)
1065 D("%s", nm_dump_buf(BDG_NMB(na, slot),len, 128, NULL));
1289 D("%s", nm_dump_buf(NMB(na, slot),len, 128, NULL));
1066
1067 slot->len = len;
1068 slot->flags = kring->nkr_slot_flags;
1069 nm_i = nm_next(nm_i, lim);
1290
1291 slot->len = len;
1292 slot->flags = kring->nkr_slot_flags;
1293 nm_i = nm_next(nm_i, lim);
1294 m_freem(m);
1070 }
1071 kring->nr_hwtail = nm_i;
1072 }
1073
1074 /*
1075 * Second part: skip past packets that userspace has released.
1076 */
1077 nm_i = kring->nr_hwcur;
1078 if (nm_i != head) { /* something was released */
1079 if (netmap_fwd || kring->ring->flags & NR_FORWARD)
1080 ret = netmap_sw_to_nic(na);
1081 kring->nr_hwcur = head;
1082 }
1083
1084 nm_rxsync_finalize(kring);
1085
1295 }
1296 kring->nr_hwtail = nm_i;
1297 }
1298
1299 /*
1300 * Second part: skip past packets that userspace has released.
1301 */
1302 nm_i = kring->nr_hwcur;
1303 if (nm_i != head) { /* something was released */
1304 if (netmap_fwd || kring->ring->flags & NR_FORWARD)
1305 ret = netmap_sw_to_nic(na);
1306 kring->nr_hwcur = head;
1307 }
1308
1309 nm_rxsync_finalize(kring);
1310
1311 /* access copies of cur,tail in the kring */
1312 if (kring->rcur == kring->rtail && td) /* no bufs available */
1313 selrecord(td, &kring->si);
1314
1086 mbq_unlock(q);
1087 return ret;
1088}
1089
1090
1091/* Get a netmap adapter for the port.
1092 *
1093 * If it is possible to satisfy the request, return 0

--- 29 unchanged lines hidden (view full) ---

1123
1124 *na = NULL; /* default */
1125
1126 /* reset in case of invalid value */
1127 if (i < NETMAP_ADMODE_BEST || i >= NETMAP_ADMODE_LAST)
1128 i = netmap_admode = NETMAP_ADMODE_BEST;
1129
1130 if (NETMAP_CAPABLE(ifp)) {
1315 mbq_unlock(q);
1316 return ret;
1317}
1318
1319
1320/* Get a netmap adapter for the port.
1321 *
1322 * If it is possible to satisfy the request, return 0

--- 29 unchanged lines hidden (view full) ---

1352
1353 *na = NULL; /* default */
1354
1355 /* reset in case of invalid value */
1356 if (i < NETMAP_ADMODE_BEST || i >= NETMAP_ADMODE_LAST)
1357 i = netmap_admode = NETMAP_ADMODE_BEST;
1358
1359 if (NETMAP_CAPABLE(ifp)) {
1131 /* If an adapter already exists, but is
1132 * attached to a vale port, we report that the
1133 * port is busy.
1134 */
1135 if (NETMAP_OWNED_BY_KERN(NA(ifp)))
1136 return EBUSY;
1137
1360 prev_na = NA(ifp);
1138 /* If an adapter already exists, return it if
1139 * there are active file descriptors or if
1140 * netmap is not forced to use generic
1141 * adapters.
1142 */
1361 /* If an adapter already exists, return it if
1362 * there are active file descriptors or if
1363 * netmap is not forced to use generic
1364 * adapters.
1365 */
1143 if (NA(ifp)->active_fds > 0 ||
1144 i != NETMAP_ADMODE_GENERIC) {
1145 *na = NA(ifp);
1366 if (NETMAP_OWNED_BY_ANY(prev_na)
1367 || i != NETMAP_ADMODE_GENERIC
1368 || prev_na->na_flags & NAF_FORCE_NATIVE
1369#ifdef WITH_PIPES
1370 /* ugly, but we cannot allow an adapter switch
1371 * if some pipe is referring to this one
1372 */
1373 || prev_na->na_next_pipe > 0
1374#endif
1375 ) {
1376 *na = prev_na;
1146 return 0;
1147 }
1148 }
1149
1150 /* If there isn't native support and netmap is not allowed
1151 * to use generic adapters, we cannot satisfy the request.
1152 */
1153 if (!NETMAP_CAPABLE(ifp) && i == NETMAP_ADMODE_NATIVE)

--- 53 unchanged lines hidden (view full) ---

1207netmap_get_na(struct nmreq *nmr, struct netmap_adapter **na, int create)
1208{
1209 struct ifnet *ifp = NULL;
1210 int error = 0;
1211 struct netmap_adapter *ret = NULL;
1212
1213 *na = NULL; /* default return value */
1214
1377 return 0;
1378 }
1379 }
1380
1381 /* If there isn't native support and netmap is not allowed
1382 * to use generic adapters, we cannot satisfy the request.
1383 */
1384 if (!NETMAP_CAPABLE(ifp) && i == NETMAP_ADMODE_NATIVE)

--- 53 unchanged lines hidden (view full) ---

1438netmap_get_na(struct nmreq *nmr, struct netmap_adapter **na, int create)
1439{
1440 struct ifnet *ifp = NULL;
1441 int error = 0;
1442 struct netmap_adapter *ret = NULL;
1443
1444 *na = NULL; /* default return value */
1445
1215 /* first try to see if this is a bridge port. */
1216 NMG_LOCK_ASSERT();
1217
1446 NMG_LOCK_ASSERT();
1447
1448 /* we cascade through all possibile types of netmap adapter.
1449 * All netmap_get_*_na() functions return an error and an na,
1450 * with the following combinations:
1451 *
1452 * error na
1453 * 0 NULL type doesn't match
1454 * !0 NULL type matches, but na creation/lookup failed
1455 * 0 !NULL type matches and na created/found
1456 * !0 !NULL impossible
1457 */
1458
1459 /* try to see if this is a monitor port */
1460 error = netmap_get_monitor_na(nmr, na, create);
1461 if (error || *na != NULL)
1462 return error;
1463
1464 /* try to see if this is a pipe port */
1218 error = netmap_get_pipe_na(nmr, na, create);
1219 if (error || *na != NULL)
1220 return error;
1221
1465 error = netmap_get_pipe_na(nmr, na, create);
1466 if (error || *na != NULL)
1467 return error;
1468
1469 /* try to see if this is a bridge port */
1222 error = netmap_get_bdg_na(nmr, na, create);
1223 if (error)
1224 return error;
1225
1226 if (*na != NULL) /* valid match in netmap_get_bdg_na() */
1227 goto pipes;
1228
1229 /*

--- 6 unchanged lines hidden (view full) ---

1236 if (ifp == NULL) {
1237 return ENXIO;
1238 }
1239
1240 error = netmap_get_hw_na(ifp, &ret);
1241 if (error)
1242 goto out;
1243
1470 error = netmap_get_bdg_na(nmr, na, create);
1471 if (error)
1472 return error;
1473
1474 if (*na != NULL) /* valid match in netmap_get_bdg_na() */
1475 goto pipes;
1476
1477 /*

--- 6 unchanged lines hidden (view full) ---

1484 if (ifp == NULL) {
1485 return ENXIO;
1486 }
1487
1488 error = netmap_get_hw_na(ifp, &ret);
1489 if (error)
1490 goto out;
1491
1244 /* Users cannot use the NIC attached to a bridge directly */
1245 if (NETMAP_OWNED_BY_KERN(ret)) {
1246 error = EBUSY;
1247 goto out;
1248 }
1249 *na = ret;
1250 netmap_adapter_get(ret);
1251
1252pipes:
1253 /*
1254 * If we are opening a pipe whose parent was not in netmap mode,
1255 * we have to allocate the pipe array now.
1256 * XXX get rid of this clumsiness (2014-03-15)

--- 182 unchanged lines hidden (view full) ---

1439int
1440netmap_ring_reinit(struct netmap_kring *kring)
1441{
1442 struct netmap_ring *ring = kring->ring;
1443 u_int i, lim = kring->nkr_num_slots - 1;
1444 int errors = 0;
1445
1446 // XXX KASSERT nm_kr_tryget
1492 *na = ret;
1493 netmap_adapter_get(ret);
1494
1495pipes:
1496 /*
1497 * If we are opening a pipe whose parent was not in netmap mode,
1498 * we have to allocate the pipe array now.
1499 * XXX get rid of this clumsiness (2014-03-15)

--- 182 unchanged lines hidden (view full) ---

1682int
1683netmap_ring_reinit(struct netmap_kring *kring)
1684{
1685 struct netmap_ring *ring = kring->ring;
1686 u_int i, lim = kring->nkr_num_slots - 1;
1687 int errors = 0;
1688
1689 // XXX KASSERT nm_kr_tryget
1447 RD(10, "called for %s", NM_IFPNAME(kring->na->ifp));
1690 RD(10, "called for %s", kring->name);
1448 // XXX probably wrong to trust userspace
1449 kring->rhead = ring->head;
1450 kring->rcur = ring->cur;
1451 kring->rtail = ring->tail;
1452
1453 if (ring->cur > lim)
1454 errors++;
1455 if (ring->head > lim)
1456 errors++;
1457 if (ring->tail > lim)
1458 errors++;
1459 for (i = 0; i <= lim; i++) {
1460 u_int idx = ring->slot[i].buf_idx;
1461 u_int len = ring->slot[i].len;
1462 if (idx < 2 || idx >= netmap_total_buffers) {
1463 RD(5, "bad index at slot %d idx %d len %d ", i, idx, len);
1464 ring->slot[i].buf_idx = 0;
1465 ring->slot[i].len = 0;
1691 // XXX probably wrong to trust userspace
1692 kring->rhead = ring->head;
1693 kring->rcur = ring->cur;
1694 kring->rtail = ring->tail;
1695
1696 if (ring->cur > lim)
1697 errors++;
1698 if (ring->head > lim)
1699 errors++;
1700 if (ring->tail > lim)
1701 errors++;
1702 for (i = 0; i <= lim; i++) {
1703 u_int idx = ring->slot[i].buf_idx;
1704 u_int len = ring->slot[i].len;
1705 if (idx < 2 || idx >= netmap_total_buffers) {
1706 RD(5, "bad index at slot %d idx %d len %d ", i, idx, len);
1707 ring->slot[i].buf_idx = 0;
1708 ring->slot[i].len = 0;
1466 } else if (len > NETMAP_BDG_BUF_SIZE(kring->na->nm_mem)) {
1709 } else if (len > NETMAP_BUF_SIZE(kring->na)) {
1467 ring->slot[i].len = 0;
1468 RD(5, "bad len at slot %d idx %d len %d", i, idx, len);
1469 }
1470 }
1471 if (errors) {
1472 RD(10, "total %d errors", errors);
1473 RD(10, "%s reinit, cur %d -> %d tail %d -> %d",
1474 kring->name,
1475 ring->cur, kring->nr_hwcur,
1476 ring->tail, kring->nr_hwtail);
1477 ring->head = kring->rhead = kring->nr_hwcur;
1478 ring->cur = kring->rcur = kring->nr_hwcur;
1479 ring->tail = kring->rtail = kring->nr_hwtail;
1480 }
1481 return (errors ? 1 : 0);
1482}
1483
1710 ring->slot[i].len = 0;
1711 RD(5, "bad len at slot %d idx %d len %d", i, idx, len);
1712 }
1713 }
1714 if (errors) {
1715 RD(10, "total %d errors", errors);
1716 RD(10, "%s reinit, cur %d -> %d tail %d -> %d",
1717 kring->name,
1718 ring->cur, kring->nr_hwcur,
1719 ring->tail, kring->nr_hwtail);
1720 ring->head = kring->rhead = kring->nr_hwcur;
1721 ring->cur = kring->rcur = kring->nr_hwcur;
1722 ring->tail = kring->rtail = kring->nr_hwtail;
1723 }
1724 return (errors ? 1 : 0);
1725}
1726
1484
1485/*
1486 * Set the ring ID. For devices with a single queue, a request
1487 * for all rings is the same as a single ring.
1727/* interpret the ringid and flags fields of an nmreq, by translating them
1728 * into a pair of intervals of ring indices:
1729 *
1730 * [priv->np_txqfirst, priv->np_txqlast) and
1731 * [priv->np_rxqfirst, priv->np_rxqlast)
1732 *
1488 */
1733 */
1489static int
1490netmap_set_ringid(struct netmap_priv_d *priv, uint16_t ringid, uint32_t flags)
1734int
1735netmap_interp_ringid(struct netmap_priv_d *priv, uint16_t ringid, uint32_t flags)
1491{
1492 struct netmap_adapter *na = priv->np_na;
1493 u_int j, i = ringid & NETMAP_RING_MASK;
1494 u_int reg = flags & NR_REG_MASK;
1495
1496 if (reg == NR_REG_DEFAULT) {
1497 /* convert from old ringid to flags */
1498 if (ringid & NETMAP_SW_RING) {

--- 47 unchanged lines hidden (view full) ---

1546 j = 0;
1547 priv->np_rxqfirst = j;
1548 priv->np_rxqlast = j + 1;
1549 break;
1550 default:
1551 D("invalid regif type %d", reg);
1552 return EINVAL;
1553 }
1736{
1737 struct netmap_adapter *na = priv->np_na;
1738 u_int j, i = ringid & NETMAP_RING_MASK;
1739 u_int reg = flags & NR_REG_MASK;
1740
1741 if (reg == NR_REG_DEFAULT) {
1742 /* convert from old ringid to flags */
1743 if (ringid & NETMAP_SW_RING) {

--- 47 unchanged lines hidden (view full) ---

1791 j = 0;
1792 priv->np_rxqfirst = j;
1793 priv->np_rxqlast = j + 1;
1794 break;
1795 default:
1796 D("invalid regif type %d", reg);
1797 return EINVAL;
1798 }
1554 priv->np_txpoll = (ringid & NETMAP_NO_TX_POLL) ? 0 : 1;
1555 priv->np_flags = (flags & ~NR_REG_MASK) | reg;
1799 priv->np_flags = (flags & ~NR_REG_MASK) | reg;
1556 if (nm_tx_si_user(priv))
1557 na->tx_si_users++;
1558 if (nm_rx_si_user(priv))
1559 na->rx_si_users++;
1800
1560 if (netmap_verbose) {
1561 D("%s: tx [%d,%d) rx [%d,%d) id %d",
1801 if (netmap_verbose) {
1802 D("%s: tx [%d,%d) rx [%d,%d) id %d",
1562 NM_IFPNAME(na->ifp),
1803 na->name,
1563 priv->np_txqfirst,
1564 priv->np_txqlast,
1565 priv->np_rxqfirst,
1566 priv->np_rxqlast,
1567 i);
1568 }
1569 return 0;
1570}
1571
1804 priv->np_txqfirst,
1805 priv->np_txqlast,
1806 priv->np_rxqfirst,
1807 priv->np_rxqlast,
1808 i);
1809 }
1810 return 0;
1811}
1812
1813
1572/*
1814/*
1815 * Set the ring ID. For devices with a single queue, a request
1816 * for all rings is the same as a single ring.
1817 */
1818static int
1819netmap_set_ringid(struct netmap_priv_d *priv, uint16_t ringid, uint32_t flags)
1820{
1821 struct netmap_adapter *na = priv->np_na;
1822 int error;
1823
1824 error = netmap_interp_ringid(priv, ringid, flags);
1825 if (error) {
1826 return error;
1827 }
1828
1829 priv->np_txpoll = (ringid & NETMAP_NO_TX_POLL) ? 0 : 1;
1830
1831 /* optimization: count the users registered for more than
1832 * one ring, which are the ones sleeping on the global queue.
1833 * The default netmap_notify() callback will then
1834 * avoid signaling the global queue if nobody is using it
1835 */
1836 if (nm_tx_si_user(priv))
1837 na->tx_si_users++;
1838 if (nm_rx_si_user(priv))
1839 na->rx_si_users++;
1840 return 0;
1841}
1842
1843/*
1573 * possibly move the interface to netmap-mode.
1574 * If success it returns a pointer to netmap_if, otherwise NULL.
1575 * This must be called with NMG_LOCK held.
1844 * possibly move the interface to netmap-mode.
1845 * If success it returns a pointer to netmap_if, otherwise NULL.
1846 * This must be called with NMG_LOCK held.
1847 *
1848 * The following na callbacks are called in the process:
1849 *
1850 * na->nm_config() [by netmap_update_config]
1851 * (get current number and size of rings)
1852 *
1853 * We have a generic one for linux (netmap_linux_config).
1854 * The bwrap has to override this, since it has to forward
1855 * the request to the wrapped adapter (netmap_bwrap_config).
1856 *
1857 * XXX netmap_if_new calls this again (2014-03-15)
1858 *
1859 * na->nm_krings_create() [by netmap_if_new]
1860 * (create and init the krings array)
1861 *
1862 * One of the following:
1863 *
1864 * * netmap_hw_krings_create, (hw ports)
1865 * creates the standard layout for the krings
1866 * and adds the mbq (used for the host rings).
1867 *
1868 * * netmap_vp_krings_create (VALE ports)
1869 * add leases and scratchpads
1870 *
1871 * * netmap_pipe_krings_create (pipes)
1872 * create the krings and rings of both ends and
1873 * cross-link them
1874 *
1875 * * netmap_monitor_krings_create (monitors)
1876 * avoid allocating the mbq
1877 *
1878 * * netmap_bwrap_krings_create (bwraps)
1879 * create both the brap krings array,
1880 * the krings array of the wrapped adapter, and
1881 * (if needed) the fake array for the host adapter
1882 *
1883 * na->nm_register(, 1)
1884 * (put the adapter in netmap mode)
1885 *
1886 * This may be one of the following:
1887 * (XXX these should be either all *_register or all *_reg 2014-03-15)
1888 *
1889 * * netmap_hw_register (hw ports)
1890 * checks that the ifp is still there, then calls
1891 * the hardware specific callback;
1892 *
1893 * * netmap_vp_reg (VALE ports)
1894 * If the port is connected to a bridge,
1895 * set the NAF_NETMAP_ON flag under the
1896 * bridge write lock.
1897 *
1898 * * netmap_pipe_reg (pipes)
1899 * inform the other pipe end that it is no
1900 * longer responsibile for the lifetime of this
1901 * pipe end
1902 *
1903 * * netmap_monitor_reg (monitors)
1904 * intercept the sync callbacks of the monitored
1905 * rings
1906 *
1907 * * netmap_bwrap_register (bwraps)
1908 * cross-link the bwrap and hwna rings,
1909 * forward the request to the hwna, override
1910 * the hwna notify callback (to get the frames
1911 * coming from outside go through the bridge).
1912 *
1913 * XXX maybe netmap_if_new() should be merged with this (2014-03-15).
1914 *
1576 */
1577struct netmap_if *
1578netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
1579 uint16_t ringid, uint32_t flags, int *err)
1580{
1915 */
1916struct netmap_if *
1917netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
1918 uint16_t ringid, uint32_t flags, int *err)
1919{
1581 struct ifnet *ifp = na->ifp;
1582 struct netmap_if *nifp = NULL;
1583 int error, need_mem = 0;
1584
1585 NMG_LOCK_ASSERT();
1586 /* ring configuration may have changed, fetch from the card */
1587 netmap_update_config(na);
1588 priv->np_na = na; /* store the reference */
1589 error = netmap_set_ringid(priv, ringid, flags);
1590 if (error)
1591 goto out;
1592 /* ensure allocators are ready */
1593 need_mem = !netmap_have_memory_locked(priv);
1594 if (need_mem) {
1595 error = netmap_get_memory_locked(priv);
1596 ND("get_memory returned %d", error);
1597 if (error)
1598 goto out;
1599 }
1920 struct netmap_if *nifp = NULL;
1921 int error, need_mem = 0;
1922
1923 NMG_LOCK_ASSERT();
1924 /* ring configuration may have changed, fetch from the card */
1925 netmap_update_config(na);
1926 priv->np_na = na; /* store the reference */
1927 error = netmap_set_ringid(priv, ringid, flags);
1928 if (error)
1929 goto out;
1930 /* ensure allocators are ready */
1931 need_mem = !netmap_have_memory_locked(priv);
1932 if (need_mem) {
1933 error = netmap_get_memory_locked(priv);
1934 ND("get_memory returned %d", error);
1935 if (error)
1936 goto out;
1937 }
1600 nifp = netmap_if_new(NM_IFPNAME(ifp), na);
1601
1602 /* Allocate a netmap_if and, if necessary, all the netmap_ring's */
1938 /* Allocate a netmap_if and, if necessary, all the netmap_ring's */
1939 nifp = netmap_if_new(na);
1603 if (nifp == NULL) { /* allocation failed */
1604 error = ENOMEM;
1605 goto out;
1606 }
1607 na->active_fds++;
1940 if (nifp == NULL) { /* allocation failed */
1941 error = ENOMEM;
1942 goto out;
1943 }
1944 na->active_fds++;
1608 if (ifp->if_capenable & IFCAP_NETMAP) {
1609 /* was already set */
1610 } else {
1611 /* Otherwise set the card in netmap mode
1945 if (!nm_netmap_on(na)) {
1946 /* Netmap not active, set the card in netmap mode
1612 * and make it use the shared buffers.
1613 */
1614 /* cache the allocator info in the na */
1947 * and make it use the shared buffers.
1948 */
1949 /* cache the allocator info in the na */
1615 na->na_lut = na->nm_mem->pools[NETMAP_BUF_POOL].lut;
1950 na->na_lut = netmap_mem_get_lut(na->nm_mem);
1616 ND("%p->na_lut == %p", na, na->na_lut);
1951 ND("%p->na_lut == %p", na, na->na_lut);
1617 na->na_lut_objtotal = na->nm_mem->pools[NETMAP_BUF_POOL].objtotal;
1952 na->na_lut_objtotal = netmap_mem_get_buftotal(na->nm_mem);
1953 na->na_lut_objsize = netmap_mem_get_bufsize(na->nm_mem);
1618 error = na->nm_register(na, 1); /* mode on */
1619 if (error) {
1620 netmap_do_unregif(priv, nifp);
1621 nifp = NULL;
1622 }
1623 }
1624out:
1625 *err = error;
1626 if (error) {
1954 error = na->nm_register(na, 1); /* mode on */
1955 if (error) {
1956 netmap_do_unregif(priv, nifp);
1957 nifp = NULL;
1958 }
1959 }
1960out:
1961 *err = error;
1962 if (error) {
1627 priv->np_na = NULL;
1628 /* we should drop the allocator, but only
1629 * if we were the ones who grabbed it
1630 */
1631 if (need_mem)
1632 netmap_drop_memory_locked(priv);
1963 /* we should drop the allocator, but only
1964 * if we were the ones who grabbed it
1965 */
1966 if (need_mem)
1967 netmap_drop_memory_locked(priv);
1968 priv->np_na = NULL;
1633 }
1634 if (nifp != NULL) {
1635 /*
1636 * advertise that the interface is ready bt setting ni_nifp.
1637 * The barrier is needed because readers (poll and *SYNC)
1638 * check for priv->np_nifp != NULL without locking
1639 */
1640 wmb(); /* make sure previous writes are visible to all CPUs */

--- 16 unchanged lines hidden (view full) ---

1657 *
1658 * Return 0 on success, errno otherwise.
1659 */
1660int
1661netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
1662 int fflag, struct thread *td)
1663{
1664 struct netmap_priv_d *priv = NULL;
1969 }
1970 if (nifp != NULL) {
1971 /*
1972 * advertise that the interface is ready bt setting ni_nifp.
1973 * The barrier is needed because readers (poll and *SYNC)
1974 * check for priv->np_nifp != NULL without locking
1975 */
1976 wmb(); /* make sure previous writes are visible to all CPUs */

--- 16 unchanged lines hidden (view full) ---

1993 *
1994 * Return 0 on success, errno otherwise.
1995 */
1996int
1997netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
1998 int fflag, struct thread *td)
1999{
2000 struct netmap_priv_d *priv = NULL;
1665 struct ifnet *ifp = NULL;
1666 struct nmreq *nmr = (struct nmreq *) data;
1667 struct netmap_adapter *na = NULL;
1668 int error;
1669 u_int i, qfirst, qlast;
1670 struct netmap_if *nifp;
1671 struct netmap_kring *krings;
1672
1673 (void)dev; /* UNUSED */

--- 61 unchanged lines hidden (view full) ---

1735 } while (0);
1736 NMG_UNLOCK();
1737 break;
1738
1739 case NIOCREGIF:
1740 /* possibly attach/detach NIC and VALE switch */
1741 i = nmr->nr_cmd;
1742 if (i == NETMAP_BDG_ATTACH || i == NETMAP_BDG_DETACH
2001 struct nmreq *nmr = (struct nmreq *) data;
2002 struct netmap_adapter *na = NULL;
2003 int error;
2004 u_int i, qfirst, qlast;
2005 struct netmap_if *nifp;
2006 struct netmap_kring *krings;
2007
2008 (void)dev; /* UNUSED */

--- 61 unchanged lines hidden (view full) ---

2070 } while (0);
2071 NMG_UNLOCK();
2072 break;
2073
2074 case NIOCREGIF:
2075 /* possibly attach/detach NIC and VALE switch */
2076 i = nmr->nr_cmd;
2077 if (i == NETMAP_BDG_ATTACH || i == NETMAP_BDG_DETACH
1743 || i == NETMAP_BDG_VNET_HDR) {
2078 || i == NETMAP_BDG_VNET_HDR
2079 || i == NETMAP_BDG_NEWIF
2080 || i == NETMAP_BDG_DELIF) {
1744 error = netmap_bdg_ctl(nmr, NULL);
1745 break;
1746 } else if (i != 0) {
1747 D("nr_cmd must be 0 not %d", i);
1748 error = EINVAL;
1749 break;
1750 }
1751

--- 5 unchanged lines hidden (view full) ---

1757 if (priv->np_na != NULL) { /* thread already registered */
1758 error = EBUSY;
1759 break;
1760 }
1761 /* find the interface and a reference */
1762 error = netmap_get_na(nmr, &na, 1 /* create */); /* keep reference */
1763 if (error)
1764 break;
2081 error = netmap_bdg_ctl(nmr, NULL);
2082 break;
2083 } else if (i != 0) {
2084 D("nr_cmd must be 0 not %d", i);
2085 error = EINVAL;
2086 break;
2087 }
2088

--- 5 unchanged lines hidden (view full) ---

2094 if (priv->np_na != NULL) { /* thread already registered */
2095 error = EBUSY;
2096 break;
2097 }
2098 /* find the interface and a reference */
2099 error = netmap_get_na(nmr, &na, 1 /* create */); /* keep reference */
2100 if (error)
2101 break;
1765 ifp = na->ifp;
1766 if (NETMAP_OWNED_BY_KERN(na)) {
1767 netmap_adapter_put(na);
1768 error = EBUSY;
1769 break;
1770 }
1771 nifp = netmap_do_regif(priv, na, nmr->nr_ringid, nmr->nr_flags, &error);
1772 if (!nifp) { /* reg. failed, release priv and ref */
1773 netmap_adapter_put(na);

--- 45 unchanged lines hidden (view full) ---

1819 na = priv->np_na; /* we have a reference */
1820
1821 if (na == NULL) {
1822 D("Internal error: nifp != NULL && na == NULL");
1823 error = ENXIO;
1824 break;
1825 }
1826
2102 if (NETMAP_OWNED_BY_KERN(na)) {
2103 netmap_adapter_put(na);
2104 error = EBUSY;
2105 break;
2106 }
2107 nifp = netmap_do_regif(priv, na, nmr->nr_ringid, nmr->nr_flags, &error);
2108 if (!nifp) { /* reg. failed, release priv and ref */
2109 netmap_adapter_put(na);

--- 45 unchanged lines hidden (view full) ---

2155 na = priv->np_na; /* we have a reference */
2156
2157 if (na == NULL) {
2158 D("Internal error: nifp != NULL && na == NULL");
2159 error = ENXIO;
2160 break;
2161 }
2162
1827 ifp = na->ifp;
1828 if (ifp == NULL) {
1829 RD(1, "the ifp is gone");
2163 if (!nm_netmap_on(na)) {
1830 error = ENXIO;
1831 break;
1832 }
1833
1834 if (cmd == NIOCTXSYNC) {
1835 krings = na->tx_rings;
1836 qfirst = priv->np_txqfirst;
1837 qlast = priv->np_txqlast;

--- 27 unchanged lines hidden (view full) ---

1865 kring->nm_sync(kring, NAF_FORCE_READ);
1866 microtime(&na->rx_rings[i].ring->ts);
1867 }
1868 nm_kr_put(kring);
1869 }
1870
1871 break;
1872
2164 error = ENXIO;
2165 break;
2166 }
2167
2168 if (cmd == NIOCTXSYNC) {
2169 krings = na->tx_rings;
2170 qfirst = priv->np_txqfirst;
2171 qlast = priv->np_txqlast;

--- 27 unchanged lines hidden (view full) ---

2199 kring->nm_sync(kring, NAF_FORCE_READ);
2200 microtime(&na->rx_rings[i].ring->ts);
2201 }
2202 nm_kr_put(kring);
2203 }
2204
2205 break;
2206
2207 case NIOCCONFIG:
2208 error = netmap_bdg_config(nmr);
2209 break;
1873#ifdef __FreeBSD__
1874 case FIONBIO:
1875 case FIOASYNC:
1876 ND("FIONBIO/FIOASYNC are no-ops");
1877 break;
1878
1879 case BIOCIMMEDIATE:
1880 case BIOCGHDRCMPLT:
1881 case BIOCSHDRCMPLT:
1882 case BIOCSSEESENT:
1883 D("ignore BIOCIMMEDIATE/BIOCSHDRCMPLT/BIOCSHDRCMPLT/BIOCSSEESENT");
1884 break;
1885
1886 default: /* allow device-specific ioctls */
1887 {
1888 struct socket so;
2210#ifdef __FreeBSD__
2211 case FIONBIO:
2212 case FIOASYNC:
2213 ND("FIONBIO/FIOASYNC are no-ops");
2214 break;
2215
2216 case BIOCIMMEDIATE:
2217 case BIOCGHDRCMPLT:
2218 case BIOCSHDRCMPLT:
2219 case BIOCSSEESENT:
2220 D("ignore BIOCIMMEDIATE/BIOCSHDRCMPLT/BIOCSHDRCMPLT/BIOCSSEESENT");
2221 break;
2222
2223 default: /* allow device-specific ioctls */
2224 {
2225 struct socket so;
2226 struct ifnet *ifp;
1889
1890 bzero(&so, sizeof(so));
1891 NMG_LOCK();
1892 error = netmap_get_na(nmr, &na, 0 /* don't create */); /* keep reference */
1893 if (error) {
1894 netmap_adapter_put(na);
1895 NMG_UNLOCK();
1896 break;

--- 33 unchanged lines hidden (view full) ---

1930 * The first one is remapped to pwait as selrecord() uses the name as an
1931 * hidden argument.
1932 */
1933int
1934netmap_poll(struct cdev *dev, int events, struct thread *td)
1935{
1936 struct netmap_priv_d *priv = NULL;
1937 struct netmap_adapter *na;
2227
2228 bzero(&so, sizeof(so));
2229 NMG_LOCK();
2230 error = netmap_get_na(nmr, &na, 0 /* don't create */); /* keep reference */
2231 if (error) {
2232 netmap_adapter_put(na);
2233 NMG_UNLOCK();
2234 break;

--- 33 unchanged lines hidden (view full) ---

2268 * The first one is remapped to pwait as selrecord() uses the name as an
2269 * hidden argument.
2270 */
2271int
2272netmap_poll(struct cdev *dev, int events, struct thread *td)
2273{
2274 struct netmap_priv_d *priv = NULL;
2275 struct netmap_adapter *na;
1938 struct ifnet *ifp;
1939 struct netmap_kring *kring;
1940 u_int i, check_all_tx, check_all_rx, want_tx, want_rx, revents = 0;
1941 struct mbq q; /* packets from hw queues to host stack */
1942 void *pwait = dev; /* linux compatibility */
1943 int is_kevent = 0;
1944
1945 /*
1946 * In order to avoid nested locks, we need to "double check"

--- 22 unchanged lines hidden (view full) ---

1969
1970 if (priv->np_nifp == NULL) {
1971 D("No if registered");
1972 return POLLERR;
1973 }
1974 rmb(); /* make sure following reads are not from cache */
1975
1976 na = priv->np_na;
2276 struct netmap_kring *kring;
2277 u_int i, check_all_tx, check_all_rx, want_tx, want_rx, revents = 0;
2278 struct mbq q; /* packets from hw queues to host stack */
2279 void *pwait = dev; /* linux compatibility */
2280 int is_kevent = 0;
2281
2282 /*
2283 * In order to avoid nested locks, we need to "double check"

--- 22 unchanged lines hidden (view full) ---

2306
2307 if (priv->np_nifp == NULL) {
2308 D("No if registered");
2309 return POLLERR;
2310 }
2311 rmb(); /* make sure following reads are not from cache */
2312
2313 na = priv->np_na;
1977 ifp = na->ifp;
1978 // check for deleted
1979 if (ifp == NULL) {
1980 RD(1, "the ifp is gone");
1981 return POLLERR;
1982 }
1983
2314
1984 if ( (ifp->if_capenable & IFCAP_NETMAP) == 0)
2315 if (!nm_netmap_on(na))
1985 return POLLERR;
1986
1987 if (netmap_verbose & 0x8000)
2316 return POLLERR;
2317
2318 if (netmap_verbose & 0x8000)
1988 D("device %s events 0x%x", NM_IFPNAME(ifp), events);
2319 D("device %s events 0x%x", na->name, events);
1989 want_tx = events & (POLLOUT | POLLWRNORM);
1990 want_rx = events & (POLLIN | POLLRDNORM);
1991
1992
1993 /*
1994 * check_all_{tx|rx} are set if the card has more than one queue AND
1995 * the file descriptor is bound to all of them. If so, we sleep on
1996 * the "global" selinfo, otherwise we sleep on individual selinfo

--- 54 unchanged lines hidden (view full) ---

2051 continue;
2052 /* only one thread does txsync */
2053 if (nm_kr_tryget(kring)) {
2054 /* either busy or stopped
2055 * XXX if the ring is stopped, sleeping would
2056 * be better. In current code, however, we only
2057 * stop the rings for brief intervals (2014-03-14)
2058 */
2320 want_tx = events & (POLLOUT | POLLWRNORM);
2321 want_rx = events & (POLLIN | POLLRDNORM);
2322
2323
2324 /*
2325 * check_all_{tx|rx} are set if the card has more than one queue AND
2326 * the file descriptor is bound to all of them. If so, we sleep on
2327 * the "global" selinfo, otherwise we sleep on individual selinfo

--- 54 unchanged lines hidden (view full) ---

2382 continue;
2383 /* only one thread does txsync */
2384 if (nm_kr_tryget(kring)) {
2385 /* either busy or stopped
2386 * XXX if the ring is stopped, sleeping would
2387 * be better. In current code, however, we only
2388 * stop the rings for brief intervals (2014-03-14)
2389 */
2059
2060 if (netmap_verbose)
2061 RD(2, "%p lost race on txring %d, ok",
2062 priv, i);
2063 continue;
2064 }
2065 if (nm_txsync_prologue(kring) >= kring->nkr_num_slots) {
2066 netmap_ring_reinit(kring);
2067 revents |= POLLERR;

--- 42 unchanged lines hidden (view full) ---

2110 RD(2, "%p lost race on rxring %d, ok",
2111 priv, i);
2112 continue;
2113 }
2114
2115 /*
2116 * transparent mode support: collect packets
2117 * from the rxring(s).
2390 if (netmap_verbose)
2391 RD(2, "%p lost race on txring %d, ok",
2392 priv, i);
2393 continue;
2394 }
2395 if (nm_txsync_prologue(kring) >= kring->nkr_num_slots) {
2396 netmap_ring_reinit(kring);
2397 revents |= POLLERR;

--- 42 unchanged lines hidden (view full) ---

2440 RD(2, "%p lost race on rxring %d, ok",
2441 priv, i);
2442 continue;
2443 }
2444
2445 /*
2446 * transparent mode support: collect packets
2447 * from the rxring(s).
2448 * XXX NR_FORWARD should only be read on
2449 * physical or NIC ports
2118 */
2119 if (netmap_fwd ||kring->ring->flags & NR_FORWARD) {
2120 ND(10, "forwarding some buffers up %d to %d",
2121 kring->nr_hwcur, kring->ring->cur);
2122 netmap_grab_packets(kring, &q, netmap_fwd);
2123 }
2124
2125 if (kring->nm_sync(kring, 0))

--- 10 unchanged lines hidden (view full) ---

2136 retry_rx = 0;
2137 na->nm_notify(na, i, NR_RX, 0);
2138 }
2139 }
2140
2141 /* transparent mode XXX only during first pass ? */
2142 if (na->na_flags & NAF_HOST_RINGS) {
2143 kring = &na->rx_rings[na->num_rx_rings];
2450 */
2451 if (netmap_fwd ||kring->ring->flags & NR_FORWARD) {
2452 ND(10, "forwarding some buffers up %d to %d",
2453 kring->nr_hwcur, kring->ring->cur);
2454 netmap_grab_packets(kring, &q, netmap_fwd);
2455 }
2456
2457 if (kring->nm_sync(kring, 0))

--- 10 unchanged lines hidden (view full) ---

2468 retry_rx = 0;
2469 na->nm_notify(na, i, NR_RX, 0);
2470 }
2471 }
2472
2473 /* transparent mode XXX only during first pass ? */
2474 if (na->na_flags & NAF_HOST_RINGS) {
2475 kring = &na->rx_rings[na->num_rx_rings];
2144 if (netmap_fwd || kring->ring->flags & NR_FORWARD) {
2145 send_down = netmap_rxsync_from_host(na, td, dev);
2146 if (send_down && (netmap_no_timestamp == 0 ||
2147 kring->ring->flags & NR_TIMESTAMP)) {
2148 microtime(&kring->ring->ts);
2149 }
2476 if (check_all_rx
2477 && (netmap_fwd || kring->ring->flags & NR_FORWARD)) {
2478 /* XXX fix to use kring fields */
2479 if (nm_ring_empty(kring->ring))
2480 send_down = netmap_rxsync_from_host(na, td, dev);
2481 if (!nm_ring_empty(kring->ring))
2482 revents |= want_rx;
2150 }
2151 }
2152
2153 if (retry_rx && !is_kevent)
2154 selrecord(td, check_all_rx ?
2155 &na->rx_si : &na->rx_rings[priv->np_rxqfirst].si);
2156 if (send_down > 0 || retry_rx) {
2157 retry_rx = 0;

--- 11 unchanged lines hidden (view full) ---

2169 *
2170 * In this mode we also scan the sw rxring, which in
2171 * turn passes packets up.
2172 *
2173 * XXX Transparent mode at the moment requires to bind all
2174 * rings to a single file descriptor.
2175 */
2176
2483 }
2484 }
2485
2486 if (retry_rx && !is_kevent)
2487 selrecord(td, check_all_rx ?
2488 &na->rx_si : &na->rx_rings[priv->np_rxqfirst].si);
2489 if (send_down > 0 || retry_rx) {
2490 retry_rx = 0;

--- 11 unchanged lines hidden (view full) ---

2502 *
2503 * In this mode we also scan the sw rxring, which in
2504 * turn passes packets up.
2505 *
2506 * XXX Transparent mode at the moment requires to bind all
2507 * rings to a single file descriptor.
2508 */
2509
2177 if (q.head)
2510 if (q.head && na->ifp != NULL)
2178 netmap_send_up(na->ifp, &q);
2179
2180 return (revents);
2181}
2182
2183
2184/*-------------------- driver support routines -------------------*/
2185

--- 33 unchanged lines hidden (view full) ---

2219 */
2220int
2221netmap_attach_common(struct netmap_adapter *na)
2222{
2223 struct ifnet *ifp = na->ifp;
2224
2225 if (na->num_tx_rings == 0 || na->num_rx_rings == 0) {
2226 D("%s: invalid rings tx %d rx %d",
2511 netmap_send_up(na->ifp, &q);
2512
2513 return (revents);
2514}
2515
2516
2517/*-------------------- driver support routines -------------------*/
2518

--- 33 unchanged lines hidden (view full) ---

2552 */
2553int
2554netmap_attach_common(struct netmap_adapter *na)
2555{
2556 struct ifnet *ifp = na->ifp;
2557
2558 if (na->num_tx_rings == 0 || na->num_rx_rings == 0) {
2559 D("%s: invalid rings tx %d rx %d",
2227 ifp->if_xname, na->num_tx_rings, na->num_rx_rings);
2560 na->name, na->num_tx_rings, na->num_rx_rings);
2228 return EINVAL;
2229 }
2561 return EINVAL;
2562 }
2230 WNA(ifp) = na;
2563 /* ifp is NULL for virtual adapters (bwrap, non-persistent VALE ports,
2564 * pipes, monitors). For bwrap we actually have a non-null ifp for
2565 * use by the external modules, but that is set after this
2566 * function has been called.
2567 * XXX this is ugly, maybe split this function in two (2014-03-14)
2568 */
2569 if (ifp != NULL) {
2570 WNA(ifp) = na;
2231
2232 /* the following is only needed for na that use the host port.
2233 * XXX do we have something similar for linux ?
2234 */
2235#ifdef __FreeBSD__
2571
2572 /* the following is only needed for na that use the host port.
2573 * XXX do we have something similar for linux ?
2574 */
2575#ifdef __FreeBSD__
2236 na->if_input = ifp->if_input; /* for netmap_send_up */
2576 na->if_input = ifp->if_input; /* for netmap_send_up */
2237#endif /* __FreeBSD__ */
2238
2577#endif /* __FreeBSD__ */
2578
2239 NETMAP_SET_CAPABLE(ifp);
2579 NETMAP_SET_CAPABLE(ifp);
2580 }
2240 if (na->nm_krings_create == NULL) {
2241 /* we assume that we have been called by a driver,
2242 * since other port types all provide their own
2243 * nm_krings_create
2244 */
2245 na->nm_krings_create = netmap_hw_krings_create;
2246 na->nm_krings_delete = netmap_hw_krings_delete;
2247 }
2248 if (na->nm_notify == NULL)
2249 na->nm_notify = netmap_notify;
2250 na->active_fds = 0;
2251
2252 if (na->nm_mem == NULL)
2581 if (na->nm_krings_create == NULL) {
2582 /* we assume that we have been called by a driver,
2583 * since other port types all provide their own
2584 * nm_krings_create
2585 */
2586 na->nm_krings_create = netmap_hw_krings_create;
2587 na->nm_krings_delete = netmap_hw_krings_delete;
2588 }
2589 if (na->nm_notify == NULL)
2590 na->nm_notify = netmap_notify;
2591 na->active_fds = 0;
2592
2593 if (na->nm_mem == NULL)
2594 /* use the global allocator */
2253 na->nm_mem = &nm_mem;
2595 na->nm_mem = &nm_mem;
2596 if (na->nm_bdg_attach == NULL)
2597 /* no special nm_bdg_attach callback. On VALE
2598 * attach, we need to interpose a bwrap
2599 */
2600 na->nm_bdg_attach = netmap_bwrap_attach;
2254 return 0;
2255}
2256
2257
2258/* standard cleanup, called by all destructors */
2259void
2260netmap_detach_common(struct netmap_adapter *na)
2261{

--- 6 unchanged lines hidden (view full) ---

2268 }
2269 netmap_pipe_dealloc(na);
2270 if (na->na_flags & NAF_MEM_OWNER)
2271 netmap_mem_private_delete(na->nm_mem);
2272 bzero(na, sizeof(*na));
2273 free(na, M_DEVBUF);
2274}
2275
2601 return 0;
2602}
2603
2604
2605/* standard cleanup, called by all destructors */
2606void
2607netmap_detach_common(struct netmap_adapter *na)
2608{

--- 6 unchanged lines hidden (view full) ---

2615 }
2616 netmap_pipe_dealloc(na);
2617 if (na->na_flags & NAF_MEM_OWNER)
2618 netmap_mem_private_delete(na->nm_mem);
2619 bzero(na, sizeof(*na));
2620 free(na, M_DEVBUF);
2621}
2622
2623/* Wrapper for the register callback provided hardware drivers.
2624 * na->ifp == NULL means the the driver module has been
2625 * unloaded, so we cannot call into it.
2626 * Note that module unloading, in our patched linux drivers,
2627 * happens under NMG_LOCK and after having stopped all the
2628 * nic rings (see netmap_detach). This provides sufficient
2629 * protection for the other driver-provied callbacks
2630 * (i.e., nm_config and nm_*xsync), that therefore don't need
2631 * to wrapped.
2632 */
2633static int
2634netmap_hw_register(struct netmap_adapter *na, int onoff)
2635{
2636 struct netmap_hw_adapter *hwna =
2637 (struct netmap_hw_adapter*)na;
2276
2638
2639 if (na->ifp == NULL)
2640 return onoff ? ENXIO : 0;
2641
2642 return hwna->nm_hw_register(na, onoff);
2643}
2644
2645
2277/*
2278 * Initialize a ``netmap_adapter`` object created by driver on attach.
2279 * We allocate a block of memory with room for a struct netmap_adapter
2280 * plus two sets of N+2 struct netmap_kring (where N is the number
2281 * of hardware rings):
2282 * krings 0..N-1 are for the hardware queues.
2283 * kring N is for the host stack queue
2284 * kring N+1 is only used for the selinfo for all queues. // XXX still true ?

--- 8 unchanged lines hidden (view full) ---

2293
2294 if (arg == NULL || ifp == NULL)
2295 goto fail;
2296 hwna = malloc(sizeof(*hwna), M_DEVBUF, M_NOWAIT | M_ZERO);
2297 if (hwna == NULL)
2298 goto fail;
2299 hwna->up = *arg;
2300 hwna->up.na_flags |= NAF_HOST_RINGS;
2646/*
2647 * Initialize a ``netmap_adapter`` object created by driver on attach.
2648 * We allocate a block of memory with room for a struct netmap_adapter
2649 * plus two sets of N+2 struct netmap_kring (where N is the number
2650 * of hardware rings):
2651 * krings 0..N-1 are for the hardware queues.
2652 * kring N is for the host stack queue
2653 * kring N+1 is only used for the selinfo for all queues. // XXX still true ?

--- 8 unchanged lines hidden (view full) ---

2662
2663 if (arg == NULL || ifp == NULL)
2664 goto fail;
2665 hwna = malloc(sizeof(*hwna), M_DEVBUF, M_NOWAIT | M_ZERO);
2666 if (hwna == NULL)
2667 goto fail;
2668 hwna->up = *arg;
2669 hwna->up.na_flags |= NAF_HOST_RINGS;
2670 strncpy(hwna->up.name, ifp->if_xname, sizeof(hwna->up.name));
2671 hwna->nm_hw_register = hwna->up.nm_register;
2672 hwna->up.nm_register = netmap_hw_register;
2301 if (netmap_attach_common(&hwna->up)) {
2302 free(hwna, M_DEVBUF);
2303 goto fail;
2304 }
2305 netmap_adapter_get(&hwna->up);
2306
2307#ifdef linux
2308 if (ifp->netdev_ops) {
2309 /* prepare a clone of the netdev ops */
2310#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28)
2311 hwna->nm_ndo.ndo_start_xmit = ifp->netdev_ops;
2312#else
2313 hwna->nm_ndo = *ifp->netdev_ops;
2314#endif
2315 }
2316 hwna->nm_ndo.ndo_start_xmit = linux_netmap_start_xmit;
2673 if (netmap_attach_common(&hwna->up)) {
2674 free(hwna, M_DEVBUF);
2675 goto fail;
2676 }
2677 netmap_adapter_get(&hwna->up);
2678
2679#ifdef linux
2680 if (ifp->netdev_ops) {
2681 /* prepare a clone of the netdev ops */
2682#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28)
2683 hwna->nm_ndo.ndo_start_xmit = ifp->netdev_ops;
2684#else
2685 hwna->nm_ndo = *ifp->netdev_ops;
2686#endif
2687 }
2688 hwna->nm_ndo.ndo_start_xmit = linux_netmap_start_xmit;
2689 if (ifp->ethtool_ops) {
2690 hwna->nm_eto = *ifp->ethtool_ops;
2691 }
2692 hwna->nm_eto.set_ringparam = linux_netmap_set_ringparam;
2693#ifdef ETHTOOL_SCHANNELS
2694 hwna->nm_eto.set_channels = linux_netmap_set_channels;
2695#endif
2696 if (arg->nm_config == NULL) {
2697 hwna->up.nm_config = netmap_linux_config;
2698 }
2317#endif /* linux */
2318
2319 D("success for %s tx %d/%d rx %d/%d queues/slots",
2699#endif /* linux */
2700
2701 D("success for %s tx %d/%d rx %d/%d queues/slots",
2320 NM_IFPNAME(ifp),
2702 hwna->up.name,
2321 hwna->up.num_tx_rings, hwna->up.num_tx_desc,
2322 hwna->up.num_rx_rings, hwna->up.num_rx_desc
2323 );
2324 return 0;
2325
2326fail:
2327 D("fail, arg %p ifp %p na %p", arg, ifp, hwna);
2328 if (ifp)

--- 59 unchanged lines hidden (view full) ---

2388
2389 NMG_LOCK();
2390 netmap_disable_all_rings(ifp);
2391 if (!netmap_adapter_put(na)) {
2392 /* someone is still using the adapter,
2393 * tell them that the interface is gone
2394 */
2395 na->ifp = NULL;
2703 hwna->up.num_tx_rings, hwna->up.num_tx_desc,
2704 hwna->up.num_rx_rings, hwna->up.num_rx_desc
2705 );
2706 return 0;
2707
2708fail:
2709 D("fail, arg %p ifp %p na %p", arg, ifp, hwna);
2710 if (ifp)

--- 59 unchanged lines hidden (view full) ---

2770
2771 NMG_LOCK();
2772 netmap_disable_all_rings(ifp);
2773 if (!netmap_adapter_put(na)) {
2774 /* someone is still using the adapter,
2775 * tell them that the interface is gone
2776 */
2777 na->ifp = NULL;
2778 // XXX also clear NAF_NATIVE_ON ?
2779 na->na_flags &= ~NAF_NETMAP_ON;
2396 /* give them a chance to notice */
2397 netmap_enable_all_rings(ifp);
2398 }
2399 NMG_UNLOCK();
2400}
2401
2402
2403/*

--- 17 unchanged lines hidden (view full) ---

2421 u_int error = ENOBUFS;
2422 struct mbq *q;
2423 int space;
2424
2425 // XXX [Linux] we do not need this lock
2426 // if we follow the down/configure/up protocol -gl
2427 // mtx_lock(&na->core_lock);
2428
2780 /* give them a chance to notice */
2781 netmap_enable_all_rings(ifp);
2782 }
2783 NMG_UNLOCK();
2784}
2785
2786
2787/*

--- 17 unchanged lines hidden (view full) ---

2805 u_int error = ENOBUFS;
2806 struct mbq *q;
2807 int space;
2808
2809 // XXX [Linux] we do not need this lock
2810 // if we follow the down/configure/up protocol -gl
2811 // mtx_lock(&na->core_lock);
2812
2429 if ( (ifp->if_capenable & IFCAP_NETMAP) == 0) {
2430 D("%s not in netmap mode anymore", NM_IFPNAME(ifp));
2813 if (!nm_netmap_on(na)) {
2814 D("%s not in netmap mode anymore", na->name);
2431 error = ENXIO;
2432 goto done;
2433 }
2434
2435 kring = &na->rx_rings[na->num_rx_rings];
2436 q = &kring->rx_queue;
2437
2438 // XXX reconsider long packets if we handle fragments
2815 error = ENXIO;
2816 goto done;
2817 }
2818
2819 kring = &na->rx_rings[na->num_rx_rings];
2820 q = &kring->rx_queue;
2821
2822 // XXX reconsider long packets if we handle fragments
2439 if (len > NETMAP_BDG_BUF_SIZE(na->nm_mem)) { /* too long for us */
2440 D("%s from_host, drop packet size %d > %d", NM_IFPNAME(ifp),
2441 len, NETMAP_BDG_BUF_SIZE(na->nm_mem));
2823 if (len > NETMAP_BUF_SIZE(na)) { /* too long for us */
2824 D("%s from_host, drop packet size %d > %d", na->name,
2825 len, NETMAP_BUF_SIZE(na));
2442 goto done;
2443 }
2444
2445 /* protect against rxsync_from_host(), netmap_sw_to_nic()
2446 * and maybe other instances of netmap_transmit (the latter
2447 * not possible on Linux).
2448 * Also avoid overflowing the queue.
2449 */
2450 mbq_lock(q);
2451
2452 space = kring->nr_hwtail - kring->nr_hwcur;
2453 if (space < 0)
2454 space += kring->nkr_num_slots;
2455 if (space + mbq_len(q) >= kring->nkr_num_slots - 1) { // XXX
2456 RD(10, "%s full hwcur %d hwtail %d qlen %d len %d m %p",
2826 goto done;
2827 }
2828
2829 /* protect against rxsync_from_host(), netmap_sw_to_nic()
2830 * and maybe other instances of netmap_transmit (the latter
2831 * not possible on Linux).
2832 * Also avoid overflowing the queue.
2833 */
2834 mbq_lock(q);
2835
2836 space = kring->nr_hwtail - kring->nr_hwcur;
2837 if (space < 0)
2838 space += kring->nkr_num_slots;
2839 if (space + mbq_len(q) >= kring->nkr_num_slots - 1) { // XXX
2840 RD(10, "%s full hwcur %d hwtail %d qlen %d len %d m %p",
2457 NM_IFPNAME(ifp), kring->nr_hwcur, kring->nr_hwtail, mbq_len(q),
2841 na->name, kring->nr_hwcur, kring->nr_hwtail, mbq_len(q),
2458 len, m);
2459 } else {
2460 mbq_enqueue(q, m);
2461 ND(10, "%s %d bufs in queue len %d m %p",
2842 len, m);
2843 } else {
2844 mbq_enqueue(q, m);
2845 ND(10, "%s %d bufs in queue len %d m %p",
2462 NM_IFPNAME(ifp), mbq_len(q), len, m);
2846 na->name, mbq_len(q), len, m);
2463 /* notify outside the lock */
2464 m = NULL;
2465 error = 0;
2466 }
2467 mbq_unlock(q);
2468
2469done:
2470 if (m)

--- 16 unchanged lines hidden (view full) ---

2487 */
2488struct netmap_slot *
2489netmap_reset(struct netmap_adapter *na, enum txrx tx, u_int n,
2490 u_int new_cur)
2491{
2492 struct netmap_kring *kring;
2493 int new_hwofs, lim;
2494
2847 /* notify outside the lock */
2848 m = NULL;
2849 error = 0;
2850 }
2851 mbq_unlock(q);
2852
2853done:
2854 if (m)

--- 16 unchanged lines hidden (view full) ---

2871 */
2872struct netmap_slot *
2873netmap_reset(struct netmap_adapter *na, enum txrx tx, u_int n,
2874 u_int new_cur)
2875{
2876 struct netmap_kring *kring;
2877 int new_hwofs, lim;
2878
2495 if (na == NULL) {
2496 D("NULL na, should not happen");
2497 return NULL; /* no netmap support here */
2498 }
2499 if (!(na->ifp->if_capenable & IFCAP_NETMAP)) {
2500 ND("interface not in netmap mode");
2879 if (!nm_native_on(na)) {
2880 ND("interface not in native netmap mode");
2501 return NULL; /* nothing to reinitialize */
2502 }
2503
2504 /* XXX note- in the new scheme, we are not guaranteed to be
2505 * under lock (e.g. when called on a device reset).
2506 * In this case, we should set a flag and do not trust too
2507 * much the values. In practice: TODO
2508 * - set a RESET flag somewhere in the kring

--- 14 unchanged lines hidden (view full) ---

2523 }
2524 lim = kring->nkr_num_slots - 1;
2525 if (new_hwofs > lim)
2526 new_hwofs -= lim + 1;
2527
2528 /* Always set the new offset value and realign the ring. */
2529 if (netmap_verbose)
2530 D("%s %s%d hwofs %d -> %d, hwtail %d -> %d",
2881 return NULL; /* nothing to reinitialize */
2882 }
2883
2884 /* XXX note- in the new scheme, we are not guaranteed to be
2885 * under lock (e.g. when called on a device reset).
2886 * In this case, we should set a flag and do not trust too
2887 * much the values. In practice: TODO
2888 * - set a RESET flag somewhere in the kring

--- 14 unchanged lines hidden (view full) ---

2903 }
2904 lim = kring->nkr_num_slots - 1;
2905 if (new_hwofs > lim)
2906 new_hwofs -= lim + 1;
2907
2908 /* Always set the new offset value and realign the ring. */
2909 if (netmap_verbose)
2910 D("%s %s%d hwofs %d -> %d, hwtail %d -> %d",
2531 NM_IFPNAME(na->ifp),
2911 na->name,
2532 tx == NR_TX ? "TX" : "RX", n,
2533 kring->nkr_hwofs, new_hwofs,
2534 kring->nr_hwtail,
2535 tx == NR_TX ? lim : kring->nr_hwtail);
2536 kring->nkr_hwofs = new_hwofs;
2537 if (tx == NR_TX) {
2538 kring->nr_hwtail = kring->nr_hwcur + lim;
2539 if (kring->nr_hwtail > lim)

--- 25 unchanged lines hidden (view full) ---

2565 *
2566 * "work_done" is non-null on the RX path, NULL for the TX path.
2567 * We rely on the OS to make sure that there is only one active
2568 * instance per queue, and that there is appropriate locking.
2569 *
2570 * The 'notify' routine depends on what the ring is attached to.
2571 * - for a netmap file descriptor, do a selwakeup on the individual
2572 * waitqueue, plus one on the global one if needed
2912 tx == NR_TX ? "TX" : "RX", n,
2913 kring->nkr_hwofs, new_hwofs,
2914 kring->nr_hwtail,
2915 tx == NR_TX ? lim : kring->nr_hwtail);
2916 kring->nkr_hwofs = new_hwofs;
2917 if (tx == NR_TX) {
2918 kring->nr_hwtail = kring->nr_hwcur + lim;
2919 if (kring->nr_hwtail > lim)

--- 25 unchanged lines hidden (view full) ---

2945 *
2946 * "work_done" is non-null on the RX path, NULL for the TX path.
2947 * We rely on the OS to make sure that there is only one active
2948 * instance per queue, and that there is appropriate locking.
2949 *
2950 * The 'notify' routine depends on what the ring is attached to.
2951 * - for a netmap file descriptor, do a selwakeup on the individual
2952 * waitqueue, plus one on the global one if needed
2573 * - for a switch, call the proper forwarding routine
2574 * - XXX more ?
2953 * (see netmap_notify)
2954 * - for a nic connected to a switch, call the proper forwarding routine
2955 * (see netmap_bwrap_intr_notify)
2575 */
2576void
2577netmap_common_irq(struct ifnet *ifp, u_int q, u_int *work_done)
2578{
2579 struct netmap_adapter *na = NA(ifp);
2580 struct netmap_kring *kring;
2581
2582 q &= NETMAP_RING_MASK;

--- 32 unchanged lines hidden (view full) ---

2615 * and return 1.
2616 *
2617 * Finally, if called on rx from an interface connected to a switch,
2618 * calls the proper forwarding routine, and return 1.
2619 */
2620int
2621netmap_rx_irq(struct ifnet *ifp, u_int q, u_int *work_done)
2622{
2956 */
2957void
2958netmap_common_irq(struct ifnet *ifp, u_int q, u_int *work_done)
2959{
2960 struct netmap_adapter *na = NA(ifp);
2961 struct netmap_kring *kring;
2962
2963 q &= NETMAP_RING_MASK;

--- 32 unchanged lines hidden (view full) ---

2996 * and return 1.
2997 *
2998 * Finally, if called on rx from an interface connected to a switch,
2999 * calls the proper forwarding routine, and return 1.
3000 */
3001int
3002netmap_rx_irq(struct ifnet *ifp, u_int q, u_int *work_done)
3003{
2623 // XXX could we check NAF_NATIVE_ON ?
2624 if (!(ifp->if_capenable & IFCAP_NETMAP))
3004 struct netmap_adapter *na = NA(ifp);
3005
3006 /*
3007 * XXX emulated netmap mode sets NAF_SKIP_INTR so
3008 * we still use the regular driver even though the previous
3009 * check fails. It is unclear whether we should use
3010 * nm_native_on() here.
3011 */
3012 if (!nm_netmap_on(na))
2625 return 0;
2626
3013 return 0;
3014
2627 if (NA(ifp)->na_flags & NAF_SKIP_INTR) {
3015 if (na->na_flags & NAF_SKIP_INTR) {
2628 ND("use regular interrupt");
2629 return 0;
2630 }
2631
2632 netmap_common_irq(ifp, q, work_done);
2633 return 1;
2634}
2635

--- 36 unchanged lines hidden (view full) ---

2672 goto fail;
2673 /* XXX could use make_dev_credv() to get error number */
2674 netmap_dev = make_dev(&netmap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0660,
2675 "netmap");
2676 if (!netmap_dev)
2677 goto fail;
2678
2679 netmap_init_bridges();
3016 ND("use regular interrupt");
3017 return 0;
3018 }
3019
3020 netmap_common_irq(ifp, q, work_done);
3021 return 1;
3022}
3023

--- 36 unchanged lines hidden (view full) ---

3060 goto fail;
3061 /* XXX could use make_dev_credv() to get error number */
3062 netmap_dev = make_dev(&netmap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0660,
3063 "netmap");
3064 if (!netmap_dev)
3065 goto fail;
3066
3067 netmap_init_bridges();
3068#ifdef __FreeBSD__
3069 nm_vi_init_index();
3070#endif
2680 printf("netmap: loaded module\n");
2681 return (0);
2682fail:
2683 netmap_fini();
2684 return (EINVAL); /* may be incorrect */
2685}
3071 printf("netmap: loaded module\n");
3072 return (0);
3073fail:
3074 netmap_fini();
3075 return (EINVAL); /* may be incorrect */
3076}