Deleted Added
sdiff udiff text old ( 246355 ) new ( 250458 )
full compact
netmap.c (246355) netmap.c (250458)
1/*
1/*
2 * Copyright (C) 2011-2012 Matteo Landi, Luigi Rizzo. All rights reserved.
2 * Copyright (C) 2011-2013 Matteo Landi, Luigi Rizzo. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the

--- 49 unchanged lines hidden (view full) ---

60#endif /* linux */
61
62#ifdef __APPLE__
63#include "osx_glue.h"
64#endif /* __APPLE__ */
65
66#ifdef __FreeBSD__
67#include <sys/cdefs.h> /* prerequisite */
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the

--- 49 unchanged lines hidden (view full) ---

60#endif /* linux */
61
62#ifdef __APPLE__
63#include "osx_glue.h"
64#endif /* __APPLE__ */
65
66#ifdef __FreeBSD__
67#include <sys/cdefs.h> /* prerequisite */
68__FBSDID("$FreeBSD: stable/9/sys/dev/netmap/netmap.c 246355 2013-02-05 09:40:31Z luigi $");
68__FBSDID("$FreeBSD: stable/9/sys/dev/netmap/netmap.c 250458 2013-05-10 16:16:33Z luigi $");
69
70#include <sys/types.h>
71#include <sys/module.h>
72#include <sys/errno.h>
73#include <sys/param.h> /* defines used in kernel.h */
74#include <sys/jail.h>
75#include <sys/kernel.h> /* types used in module initialization */
76#include <sys/conf.h> /* cdevsw struct */
77#include <sys/uio.h> /* uio struct */
78#include <sys/sockio.h>
79#include <sys/socketvar.h> /* struct socket */
80#include <sys/malloc.h>
81#include <sys/mman.h> /* PROT_EXEC */
82#include <sys/poll.h>
83#include <sys/proc.h>
69
70#include <sys/types.h>
71#include <sys/module.h>
72#include <sys/errno.h>
73#include <sys/param.h> /* defines used in kernel.h */
74#include <sys/jail.h>
75#include <sys/kernel.h> /* types used in module initialization */
76#include <sys/conf.h> /* cdevsw struct */
77#include <sys/uio.h> /* uio struct */
78#include <sys/sockio.h>
79#include <sys/socketvar.h> /* struct socket */
80#include <sys/malloc.h>
81#include <sys/mman.h> /* PROT_EXEC */
82#include <sys/poll.h>
83#include <sys/proc.h>
84#include <sys/rwlock.h>
84#include <vm/vm.h> /* vtophys */
85#include <vm/pmap.h> /* vtophys */
86#include <sys/socket.h> /* sockaddrs */
87#include <machine/bus.h>
88#include <sys/selinfo.h>
89#include <sys/sysctl.h>
90#include <net/if.h>
91#include <net/bpf.h> /* BIOCIMMEDIATE */
92#include <net/vnet.h>
93#include <machine/bus.h> /* bus_dmamap_* */
94
95MALLOC_DEFINE(M_NETMAP, "netmap", "Network memory map");
96#endif /* __FreeBSD__ */
97
98#include <net/netmap.h>
99#include <dev/netmap/netmap_kern.h>
100
85#include <vm/vm.h> /* vtophys */
86#include <vm/pmap.h> /* vtophys */
87#include <sys/socket.h> /* sockaddrs */
88#include <machine/bus.h>
89#include <sys/selinfo.h>
90#include <sys/sysctl.h>
91#include <net/if.h>
92#include <net/bpf.h> /* BIOCIMMEDIATE */
93#include <net/vnet.h>
94#include <machine/bus.h> /* bus_dmamap_* */
95
96MALLOC_DEFINE(M_NETMAP, "netmap", "Network memory map");
97#endif /* __FreeBSD__ */
98
99#include <net/netmap.h>
100#include <dev/netmap/netmap_kern.h>
101
102/* XXX the following variables must be deprecated and included in nm_mem */
101u_int netmap_total_buffers;
102u_int netmap_buf_size;
103char *netmap_buffer_base; /* address of an invalid buffer */
104
105/* user-controlled variables */
106int netmap_verbose;
107
108static int netmap_no_timestamp; /* don't timestamp on rxsync */

--- 7 unchanged lines hidden (view full) ---

116SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, "");
117int netmap_no_pendintr = 1;
118SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr,
119 CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets.");
120
121int netmap_drop = 0; /* debugging */
122int netmap_flags = 0; /* debug flags */
123int netmap_fwd = 0; /* force transparent mode */
103u_int netmap_total_buffers;
104u_int netmap_buf_size;
105char *netmap_buffer_base; /* address of an invalid buffer */
106
107/* user-controlled variables */
108int netmap_verbose;
109
110static int netmap_no_timestamp; /* don't timestamp on rxsync */

--- 7 unchanged lines hidden (view full) ---

118SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, "");
119int netmap_no_pendintr = 1;
120SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr,
121 CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets.");
122
123int netmap_drop = 0; /* debugging */
124int netmap_flags = 0; /* debug flags */
125int netmap_fwd = 0; /* force transparent mode */
124int netmap_copy = 0; /* debugging, copy content */
125
126SYSCTL_INT(_dev_netmap, OID_AUTO, drop, CTLFLAG_RW, &netmap_drop, 0 , "");
127SYSCTL_INT(_dev_netmap, OID_AUTO, flags, CTLFLAG_RW, &netmap_flags, 0 , "");
128SYSCTL_INT(_dev_netmap, OID_AUTO, fwd, CTLFLAG_RW, &netmap_fwd, 0 , "");
126
127SYSCTL_INT(_dev_netmap, OID_AUTO, drop, CTLFLAG_RW, &netmap_drop, 0 , "");
128SYSCTL_INT(_dev_netmap, OID_AUTO, flags, CTLFLAG_RW, &netmap_flags, 0 , "");
129SYSCTL_INT(_dev_netmap, OID_AUTO, fwd, CTLFLAG_RW, &netmap_fwd, 0 , "");
129SYSCTL_INT(_dev_netmap, OID_AUTO, copy, CTLFLAG_RW, &netmap_copy, 0 , "");
130
131#ifdef NM_BRIDGE /* support for netmap bridge */
132
133/*
134 * system parameters.
135 *
136 * All switched ports have prefix NM_NAME.
137 * The switch has a max of NM_BDG_MAXPORTS ports (often stored in a bitmap,

--- 4 unchanged lines hidden (view full) ---

142 * faster. The batch size is NM_BDG_BATCH
143 */
144#define NM_NAME "vale" /* prefix for the interface */
145#define NM_BDG_MAXPORTS 16 /* up to 64 ? */
146#define NM_BRIDGE_RINGSIZE 1024 /* in the device */
147#define NM_BDG_HASH 1024 /* forwarding table entries */
148#define NM_BDG_BATCH 1024 /* entries in the forwarding buffer */
149#define NM_BRIDGES 4 /* number of bridges */
130
131#ifdef NM_BRIDGE /* support for netmap bridge */
132
133/*
134 * system parameters.
135 *
136 * All switched ports have prefix NM_NAME.
137 * The switch has a max of NM_BDG_MAXPORTS ports (often stored in a bitmap,

--- 4 unchanged lines hidden (view full) ---

142 * faster. The batch size is NM_BDG_BATCH
143 */
144#define NM_NAME "vale" /* prefix for the interface */
145#define NM_BDG_MAXPORTS 16 /* up to 64 ? */
146#define NM_BRIDGE_RINGSIZE 1024 /* in the device */
147#define NM_BDG_HASH 1024 /* forwarding table entries */
148#define NM_BDG_BATCH 1024 /* entries in the forwarding buffer */
149#define NM_BRIDGES 4 /* number of bridges */
150
151
150int netmap_bridge = NM_BDG_BATCH; /* bridge batch size */
151SYSCTL_INT(_dev_netmap, OID_AUTO, bridge, CTLFLAG_RW, &netmap_bridge, 0 , "");
152
153#ifdef linux
152int netmap_bridge = NM_BDG_BATCH; /* bridge batch size */
153SYSCTL_INT(_dev_netmap, OID_AUTO, bridge, CTLFLAG_RW, &netmap_bridge, 0 , "");
154
155#ifdef linux
154#define ADD_BDG_REF(ifp) (NA(ifp)->if_refcount++)
155#define DROP_BDG_REF(ifp) (NA(ifp)->if_refcount-- <= 1)
156
157#define refcount_acquire(_a) atomic_add(1, (atomic_t *)_a)
158#define refcount_release(_a) atomic_dec_and_test((atomic_t *)_a)
159
156#else /* !linux */
160#else /* !linux */
157#define ADD_BDG_REF(ifp) (ifp)->if_refcount++
158#define DROP_BDG_REF(ifp) refcount_release(&(ifp)->if_refcount)
161
159#ifdef __FreeBSD__
160#include <sys/endian.h>
161#include <sys/refcount.h>
162#endif /* __FreeBSD__ */
162#ifdef __FreeBSD__
163#include <sys/endian.h>
164#include <sys/refcount.h>
165#endif /* __FreeBSD__ */
166
163#define prefetch(x) __builtin_prefetch(x)
167#define prefetch(x) __builtin_prefetch(x)
168
164#endif /* !linux */
165
169#endif /* !linux */
170
171/*
172 * These are used to handle reference counters for bridge ports.
173 */
174#define ADD_BDG_REF(ifp) refcount_acquire(&NA(ifp)->na_bdg_refcount)
175#define DROP_BDG_REF(ifp) refcount_release(&NA(ifp)->na_bdg_refcount)
176
166static void bdg_netmap_attach(struct ifnet *ifp);
167static int bdg_netmap_reg(struct ifnet *ifp, int onoff);
168/* per-tx-queue entry */
169struct nm_bdg_fwd { /* forwarding entry for a bridge */
170 void *buf;
171 uint64_t dst; /* dst mask */
172 uint32_t src; /* src index ? */
173 uint16_t len; /* src len */
174};
175
176struct nm_hash_ent {
177 uint64_t mac; /* the top 2 bytes are the epoch */
178 uint64_t ports;
179};
180
181/*
177static void bdg_netmap_attach(struct ifnet *ifp);
178static int bdg_netmap_reg(struct ifnet *ifp, int onoff);
179/* per-tx-queue entry */
180struct nm_bdg_fwd { /* forwarding entry for a bridge */
181 void *buf;
182 uint64_t dst; /* dst mask */
183 uint32_t src; /* src index ? */
184 uint16_t len; /* src len */
185};
186
187struct nm_hash_ent {
188 uint64_t mac; /* the top 2 bytes are the epoch */
189 uint64_t ports;
190};
191
192/*
182 * Interfaces for a bridge are all in ports[].
193 * Interfaces for a bridge are all in bdg_ports[].
183 * The array has fixed size, an empty entry does not terminate
194 * The array has fixed size, an empty entry does not terminate
184 * the search.
195 * the search. But lookups only occur on attach/detach so we
196 * don't mind if they are slow.
197 *
198 * The bridge is non blocking on the transmit ports.
199 *
200 * bdg_lock protects accesses to the bdg_ports array.
185 */
186struct nm_bridge {
187 struct ifnet *bdg_ports[NM_BDG_MAXPORTS];
188 int n_ports;
189 uint64_t act_ports;
190 int freelist; /* first buffer index */
191 NM_SELINFO_T si; /* poll/select wait queue */
192 NM_LOCK_T bdg_lock; /* protect the selinfo ? */

--- 99 unchanged lines hidden (view full) ---

292 if (na->nm_config) {
293 na->nm_config(ifp, &txr, &txd, &rxr, &rxd);
294 } else {
295 /* take whatever we had at init time */
296 txr = na->num_tx_rings;
297 txd = na->num_tx_desc;
298 rxr = na->num_rx_rings;
299 rxd = na->num_rx_desc;
201 */
202struct nm_bridge {
203 struct ifnet *bdg_ports[NM_BDG_MAXPORTS];
204 int n_ports;
205 uint64_t act_ports;
206 int freelist; /* first buffer index */
207 NM_SELINFO_T si; /* poll/select wait queue */
208 NM_LOCK_T bdg_lock; /* protect the selinfo ? */

--- 99 unchanged lines hidden (view full) ---

308 if (na->nm_config) {
309 na->nm_config(ifp, &txr, &txd, &rxr, &rxd);
310 } else {
311 /* take whatever we had at init time */
312 txr = na->num_tx_rings;
313 txd = na->num_tx_desc;
314 rxr = na->num_rx_rings;
315 rxd = na->num_rx_desc;
300 }
316 }
301
302 if (na->num_tx_rings == txr && na->num_tx_desc == txd &&
303 na->num_rx_rings == rxr && na->num_rx_desc == rxd)
304 return 0; /* nothing changed */
305 if (netmap_verbose || na->refcount > 0) {
306 D("stored config %s: txring %d x %d, rxring %d x %d",
307 ifp->if_xname,
308 na->num_tx_rings, na->num_tx_desc,

--- 9 unchanged lines hidden (view full) ---

318 na->num_rx_desc = rxd;
319 return 0;
320 }
321 D("configuration changed while active, this is bad...");
322 return 1;
323}
324
325/*------------- memory allocator -----------------*/
317
318 if (na->num_tx_rings == txr && na->num_tx_desc == txd &&
319 na->num_rx_rings == rxr && na->num_rx_desc == rxd)
320 return 0; /* nothing changed */
321 if (netmap_verbose || na->refcount > 0) {
322 D("stored config %s: txring %d x %d, rxring %d x %d",
323 ifp->if_xname,
324 na->num_tx_rings, na->num_tx_desc,

--- 9 unchanged lines hidden (view full) ---

334 na->num_rx_desc = rxd;
335 return 0;
336 }
337 D("configuration changed while active, this is bad...");
338 return 1;
339}
340
341/*------------- memory allocator -----------------*/
326#ifdef NETMAP_MEM2
327#include "netmap_mem2.c"
342#include "netmap_mem2.c"
328#else /* !NETMAP_MEM2 */
329#include "netmap_mem1.c"
330#endif /* !NETMAP_MEM2 */
331/*------------ end of memory allocator ----------*/
332
333
334/* Structure associated to each thread which registered an interface.
335 *
336 * The first 4 fields of this structure are written by NIOCREGIF and
337 * read by poll() and NIOC?XSYNC.
338 * There is low contention among writers (actually, a correct user program

--- 153 unchanged lines hidden (view full) ---

492#endif /* NM_BRIDGE */
493}
494
495static void
496netmap_dtor(void *data)
497{
498 struct netmap_priv_d *priv = data;
499 struct ifnet *ifp = priv->np_ifp;
343/*------------ end of memory allocator ----------*/
344
345
346/* Structure associated to each thread which registered an interface.
347 *
348 * The first 4 fields of this structure are written by NIOCREGIF and
349 * read by poll() and NIOC?XSYNC.
350 * There is low contention among writers (actually, a correct user program

--- 153 unchanged lines hidden (view full) ---

504#endif /* NM_BRIDGE */
505}
506
507static void
508netmap_dtor(void *data)
509{
510 struct netmap_priv_d *priv = data;
511 struct ifnet *ifp = priv->np_ifp;
500 struct netmap_adapter *na;
501
502 NMA_LOCK();
503 if (ifp) {
512
513 NMA_LOCK();
514 if (ifp) {
504 na = NA(ifp);
515 struct netmap_adapter *na = NA(ifp);
516
505 na->nm_lock(ifp, NETMAP_REG_LOCK, 0);
506 netmap_dtor_locked(data);
507 na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0);
508
517 na->nm_lock(ifp, NETMAP_REG_LOCK, 0);
518 netmap_dtor_locked(data);
519 na->nm_lock(ifp, NETMAP_REG_UNLOCK, 0);
520
509 nm_if_rele(ifp);
521 nm_if_rele(ifp); /* might also destroy *na */
510 }
511 if (priv->ref_done) {
512 netmap_memory_deref();
513 }
514 NMA_UNLOCK();
515 bzero(priv, sizeof(*priv)); /* XXX for safety */
516 free(priv, M_DEVBUF);
517}

--- 1145 unchanged lines hidden (view full) ---

1663 na->num_rx_rings = num_queues;
1664 na->refcount = na->na_single = na->na_multi = 0;
1665 /* Core lock initialized here, others after netmap_if_new. */
1666 mtx_init(&na->core_lock, "netmap core lock", MTX_NETWORK_LOCK, MTX_DEF);
1667 if (na->nm_lock == NULL) {
1668 ND("using default locks for %s", ifp->if_xname);
1669 na->nm_lock = netmap_lock_wrapper;
1670 }
522 }
523 if (priv->ref_done) {
524 netmap_memory_deref();
525 }
526 NMA_UNLOCK();
527 bzero(priv, sizeof(*priv)); /* XXX for safety */
528 free(priv, M_DEVBUF);
529}

--- 1145 unchanged lines hidden (view full) ---

1675 na->num_rx_rings = num_queues;
1676 na->refcount = na->na_single = na->na_multi = 0;
1677 /* Core lock initialized here, others after netmap_if_new. */
1678 mtx_init(&na->core_lock, "netmap core lock", MTX_NETWORK_LOCK, MTX_DEF);
1679 if (na->nm_lock == NULL) {
1680 ND("using default locks for %s", ifp->if_xname);
1681 na->nm_lock = netmap_lock_wrapper;
1682 }
1683
1671#ifdef linux
1684#ifdef linux
1672 if (ifp->netdev_ops) {
1673 ND("netdev_ops %p", ifp->netdev_ops);
1674 /* prepare a clone of the netdev ops */
1675 na->nm_ndo = *ifp->netdev_ops;
1685 if (!ifp->netdev_ops) {
1686 D("ouch, we cannot override netdev_ops");
1687 goto fail;
1676 }
1688 }
1689#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
1690 /* if needed, prepare a clone of the entire netdev ops */
1691 na->nm_ndo = *ifp->netdev_ops;
1692#endif /* 2.6.28 and above */
1677 na->nm_ndo.ndo_start_xmit = linux_netmap_start;
1693 na->nm_ndo.ndo_start_xmit = linux_netmap_start;
1678#endif
1694#endif /* linux */
1695
1679 D("success for %s", ifp->if_xname);
1680 return 0;
1681
1682fail:
1683 D("fail, arg %p ifp %p na %p", arg, ifp, na);
1696 D("success for %s", ifp->if_xname);
1697 return 0;
1698
1699fail:
1700 D("fail, arg %p ifp %p na %p", arg, ifp, na);
1701 netmap_detach(ifp);
1684 return (na ? EINVAL : ENOMEM);
1685}
1686
1687
1688/*
1689 * Free the allocated memory linked to the given ``netmap_adapter``
1690 * object.
1691 */

--- 29 unchanged lines hidden (view full) ---

1721 struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];
1722 u_int i, len = MBUF_LEN(m);
1723 u_int error = EBUSY, lim = kring->nkr_num_slots - 1;
1724 struct netmap_slot *slot;
1725
1726 if (netmap_verbose & NM_VERB_HOST)
1727 D("%s packet %d len %d from the stack", ifp->if_xname,
1728 kring->nr_hwcur + kring->nr_hwavail, len);
1702 return (na ? EINVAL : ENOMEM);
1703}
1704
1705
1706/*
1707 * Free the allocated memory linked to the given ``netmap_adapter``
1708 * object.
1709 */

--- 29 unchanged lines hidden (view full) ---

1739 struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];
1740 u_int i, len = MBUF_LEN(m);
1741 u_int error = EBUSY, lim = kring->nkr_num_slots - 1;
1742 struct netmap_slot *slot;
1743
1744 if (netmap_verbose & NM_VERB_HOST)
1745 D("%s packet %d len %d from the stack", ifp->if_xname,
1746 kring->nr_hwcur + kring->nr_hwavail, len);
1747 if (len > NETMAP_BUF_SIZE) { /* too long for us */
1748 D("%s from_host, drop packet size %d > %d", ifp->if_xname,
1749 len, NETMAP_BUF_SIZE);
1750 m_freem(m);
1751 return EINVAL;
1752 }
1729 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
1730 if (kring->nr_hwavail >= lim) {
1731 if (netmap_verbose)
1732 D("stack ring %s full\n", ifp->if_xname);
1733 goto done; /* no space */
1734 }
1753 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
1754 if (kring->nr_hwavail >= lim) {
1755 if (netmap_verbose)
1756 D("stack ring %s full\n", ifp->if_xname);
1757 goto done; /* no space */
1758 }
1735 if (len > NETMAP_BUF_SIZE) {
1736 D("%s from_host, drop packet size %d > %d", ifp->if_xname,
1737 len, NETMAP_BUF_SIZE);
1738 goto done; /* too long for us */
1739 }
1740
1741 /* compute the insert position */
1742 i = kring->nr_hwcur + kring->nr_hwavail;
1743 if (i > lim)
1744 i -= lim + 1;
1745 slot = &kring->ring->slot[i];
1746 m_copydata(m, 0, len, NMB(slot));
1747 slot->len = len;

--- 84 unchanged lines hidden (view full) ---

1832 * lock(core); wake(i=0); unlock(core)
1833 * N rings, single lock:
1834 * lock(core); wake(i); wake(N+1) unlock(core)
1835 * 1 ring, separate locks: (i=0)
1836 * lock(i); wake(i); unlock(i)
1837 * N rings, separate locks:
1838 * lock(i); wake(i); unlock(i); lock(core) wake(N+1) unlock(core)
1839 * work_done is non-null on the RX path.
1759
1760 /* compute the insert position */
1761 i = kring->nr_hwcur + kring->nr_hwavail;
1762 if (i > lim)
1763 i -= lim + 1;
1764 slot = &kring->ring->slot[i];
1765 m_copydata(m, 0, len, NMB(slot));
1766 slot->len = len;

--- 84 unchanged lines hidden (view full) ---

1851 * lock(core); wake(i=0); unlock(core)
1852 * N rings, single lock:
1853 * lock(core); wake(i); wake(N+1) unlock(core)
1854 * 1 ring, separate locks: (i=0)
1855 * lock(i); wake(i); unlock(i)
1856 * N rings, separate locks:
1857 * lock(i); wake(i); unlock(i); lock(core) wake(N+1) unlock(core)
1858 * work_done is non-null on the RX path.
1859 *
1860 * The 'q' argument also includes flag to tell whether the queue is
1861 * already locked on enter, and whether it should remain locked on exit.
1862 * This helps adapting to different defaults in drivers and OSes.
1840 */
1841int
1842netmap_rx_irq(struct ifnet *ifp, int q, int *work_done)
1843{
1844 struct netmap_adapter *na;
1845 struct netmap_kring *r;
1846 NM_SELINFO_T *main_wq;
1863 */
1864int
1865netmap_rx_irq(struct ifnet *ifp, int q, int *work_done)
1866{
1867 struct netmap_adapter *na;
1868 struct netmap_kring *r;
1869 NM_SELINFO_T *main_wq;
1870 int locktype, unlocktype, lock;
1847
1848 if (!(ifp->if_capenable & IFCAP_NETMAP))
1849 return 0;
1871
1872 if (!(ifp->if_capenable & IFCAP_NETMAP))
1873 return 0;
1874
1875 lock = q & (NETMAP_LOCKED_ENTER | NETMAP_LOCKED_EXIT);
1876 q = q & NETMAP_RING_MASK;
1877
1850 ND(5, "received %s queue %d", work_done ? "RX" : "TX" , q);
1851 na = NA(ifp);
1852 if (na->na_flags & NAF_SKIP_INTR) {
1853 ND("use regular interrupt");
1854 return 0;
1855 }
1856
1857 if (work_done) { /* RX path */
1858 if (q >= na->num_rx_rings)
1878 ND(5, "received %s queue %d", work_done ? "RX" : "TX" , q);
1879 na = NA(ifp);
1880 if (na->na_flags & NAF_SKIP_INTR) {
1881 ND("use regular interrupt");
1882 return 0;
1883 }
1884
1885 if (work_done) { /* RX path */
1886 if (q >= na->num_rx_rings)
1859 return 0; // regular queue
1887 return 0; // not a physical queue
1860 r = na->rx_rings + q;
1861 r->nr_kflags |= NKR_PENDINTR;
1862 main_wq = (na->num_rx_rings > 1) ? &na->rx_si : NULL;
1888 r = na->rx_rings + q;
1889 r->nr_kflags |= NKR_PENDINTR;
1890 main_wq = (na->num_rx_rings > 1) ? &na->rx_si : NULL;
1863 } else { /* tx path */
1891 locktype = NETMAP_RX_LOCK;
1892 unlocktype = NETMAP_RX_UNLOCK;
1893 } else { /* TX path */
1864 if (q >= na->num_tx_rings)
1894 if (q >= na->num_tx_rings)
1865 return 0; // regular queue
1895 return 0; // not a physical queue
1866 r = na->tx_rings + q;
1867 main_wq = (na->num_tx_rings > 1) ? &na->tx_si : NULL;
1868 work_done = &q; /* dummy */
1896 r = na->tx_rings + q;
1897 main_wq = (na->num_tx_rings > 1) ? &na->tx_si : NULL;
1898 work_done = &q; /* dummy */
1899 locktype = NETMAP_TX_LOCK;
1900 unlocktype = NETMAP_TX_UNLOCK;
1869 }
1870 if (na->separate_locks) {
1901 }
1902 if (na->separate_locks) {
1871 mtx_lock(&r->q_lock);
1903 if (!(lock & NETMAP_LOCKED_ENTER))
1904 na->nm_lock(ifp, locktype, q);
1872 selwakeuppri(&r->si, PI_NET);
1905 selwakeuppri(&r->si, PI_NET);
1873 mtx_unlock(&r->q_lock);
1906 na->nm_lock(ifp, unlocktype, q);
1874 if (main_wq) {
1907 if (main_wq) {
1875 mtx_lock(&na->core_lock);
1908 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
1876 selwakeuppri(main_wq, PI_NET);
1909 selwakeuppri(main_wq, PI_NET);
1877 mtx_unlock(&na->core_lock);
1910 na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0);
1878 }
1911 }
1912 /* lock the queue again if requested */
1913 if (lock & NETMAP_LOCKED_EXIT)
1914 na->nm_lock(ifp, locktype, q);
1879 } else {
1915 } else {
1880 mtx_lock(&na->core_lock);
1916 if (!(lock & NETMAP_LOCKED_ENTER))
1917 na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
1881 selwakeuppri(&r->si, PI_NET);
1882 if (main_wq)
1883 selwakeuppri(main_wq, PI_NET);
1918 selwakeuppri(&r->si, PI_NET);
1919 if (main_wq)
1920 selwakeuppri(main_wq, PI_NET);
1884 mtx_unlock(&na->core_lock);
1921 if (!(lock & NETMAP_LOCKED_EXIT))
1922 na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0);
1885 }
1886 *work_done = 1; /* do not fire napi again */
1887 return 1;
1888}
1889
1890
1891#ifdef linux /* linux-specific routines */
1892

--- 4 unchanged lines hidden (view full) ---

1897 * events but they are filtered upstream.
1898 * If pwait != NULL, then pwait->key contains the list of events.
1899 * - events is computed from pwait as above.
1900 * - file is passed as 'td';
1901 */
1902static u_int
1903linux_netmap_poll(struct file * file, struct poll_table_struct *pwait)
1904{
1923 }
1924 *work_done = 1; /* do not fire napi again */
1925 return 1;
1926}
1927
1928
1929#ifdef linux /* linux-specific routines */
1930

--- 4 unchanged lines hidden (view full) ---

1935 * events but they are filtered upstream.
1936 * If pwait != NULL, then pwait->key contains the list of events.
1937 * - events is computed from pwait as above.
1938 * - file is passed as 'td';
1939 */
1940static u_int
1941linux_netmap_poll(struct file * file, struct poll_table_struct *pwait)
1942{
1905#if LINUX_VERSION_CODE < KERNEL_VERSION(3,4,0)
1943#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,28)
1944 int events = POLLIN | POLLOUT; /* XXX maybe... */
1945#elif LINUX_VERSION_CODE < KERNEL_VERSION(3,4,0)
1906 int events = pwait ? pwait->key : POLLIN | POLLOUT;
1907#else /* in 3.4.0 field 'key' was renamed to '_key' */
1908 int events = pwait ? pwait->_key : POLLIN | POLLOUT;
1909#endif
1910 return netmap_poll((void *)pwait, events, (void *)file);
1911}
1912
1913static int

--- 23 unchanged lines hidden (view full) ---

1937 const struct netmap_obj_pool *p = &nm_mem.pools[i];
1938 /*
1939 * In each pool memory is allocated in clusters
1940 * of size _clustsize, each containing clustentries
1941 * entries. For each object k we already store the
1942 * vtophys mapping in lut[k] so we use that, scanning
1943 * the lut[] array in steps of clustentries,
1944 * and we map each cluster (not individual pages,
1946 int events = pwait ? pwait->key : POLLIN | POLLOUT;
1947#else /* in 3.4.0 field 'key' was renamed to '_key' */
1948 int events = pwait ? pwait->_key : POLLIN | POLLOUT;
1949#endif
1950 return netmap_poll((void *)pwait, events, (void *)file);
1951}
1952
1953static int

--- 23 unchanged lines hidden (view full) ---

1977 const struct netmap_obj_pool *p = &nm_mem.pools[i];
1978 /*
1979 * In each pool memory is allocated in clusters
1980 * of size _clustsize, each containing clustentries
1981 * entries. For each object k we already store the
1982 * vtophys mapping in lut[k] so we use that, scanning
1983 * the lut[] array in steps of clustentries,
1984 * and we map each cluster (not individual pages,
1945 * it would be overkill).
1985 * it would be overkill -- XXX slow ? 20130415).
1946 */
1947
1948 /*
1949 * We interpret vm_pgoff as an offset into the whole
1950 * netmap memory, as if all clusters where contiguous.
1951 */
1952 for (lut_skip = 0, j = 0; j < p->_numclusters; j++, lut_skip += p->clustentries) {
1953 unsigned long paddr, mapsize;

--- 561 unchanged lines hidden --- 		1986 */
1987
1988 /*
1989 * We interpret vm_pgoff as an offset into the whole
1990 * netmap memory, as if all clusters where contiguous.
1991 */
1992 for (lut_skip = 0, j = 0; j < p->_numclusters; j++, lut_skip += p->clustentries) {
1993 unsigned long paddr, mapsize;

--- 561 unchanged lines hidden ---