1262153Sluigi/* 2262153Sluigi * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. 3262153Sluigi * 4262153Sluigi * Redistribution and use in source and binary forms, with or without 5262153Sluigi * modification, are permitted provided that the following conditions 6262153Sluigi * are met: 7262153Sluigi * 1. Redistributions of source code must retain the above copyright 8262153Sluigi * notice, this list of conditions and the following disclaimer. 9262153Sluigi * 2. Redistributions in binary form must reproduce the above copyright 10262153Sluigi * notice, this list of conditions and the following disclaimer in the 11262153Sluigi * documentation and/or other materials provided with the distribution. 12262153Sluigi * 13262153Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14262153Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15262153Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16262153Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17262153Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18262153Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19262153Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20262153Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21262153Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22262153Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23262153Sluigi * SUCH DAMAGE. 24262153Sluigi */ 25262153Sluigi 26262153Sluigi/* $FreeBSD$ */ 27262153Sluigi 28262153Sluigi#include <sys/types.h> 29262153Sluigi#include <sys/module.h> 30262153Sluigi#include <sys/errno.h> 31262153Sluigi#include <sys/param.h> /* defines used in kernel.h */ 32262153Sluigi#include <sys/poll.h> /* POLLIN, POLLOUT */ 33262153Sluigi#include <sys/kernel.h> /* types used in module initialization */ 34262153Sluigi#include <sys/conf.h> /* DEV_MODULE */ 35262153Sluigi#include <sys/endian.h> 36262153Sluigi 37262153Sluigi#include <sys/rwlock.h> 38262153Sluigi 39262153Sluigi#include <vm/vm.h> /* vtophys */ 40262153Sluigi#include <vm/pmap.h> /* vtophys */ 41262153Sluigi#include <vm/vm_param.h> 42262153Sluigi#include <vm/vm_object.h> 43262153Sluigi#include <vm/vm_page.h> 44262153Sluigi#include <vm/vm_pager.h> 45262153Sluigi#include <vm/uma.h> 46262153Sluigi 47262153Sluigi 48262153Sluigi#include <sys/malloc.h> 49262153Sluigi#include <sys/socket.h> /* sockaddrs */ 50262153Sluigi#include <sys/selinfo.h> 51262153Sluigi#include <net/if.h> 52262153Sluigi#include <net/if_var.h> 53262153Sluigi#include <machine/bus.h> /* bus_dmamap_* */ 54262153Sluigi#include <netinet/in.h> /* in6_cksum_pseudo() */ 55262153Sluigi#include <machine/in_cksum.h> /* in_pseudo(), in_cksum_hdr() */ 56262153Sluigi 57262153Sluigi#include <net/netmap.h> 58262153Sluigi#include <dev/netmap/netmap_kern.h> 59262153Sluigi#include <dev/netmap/netmap_mem2.h> 60262153Sluigi 61262153Sluigi 62262153Sluigi/* ======================== FREEBSD-SPECIFIC ROUTINES ================== */ 63262153Sluigi 64262153Sluigirawsum_t nm_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum) 65262153Sluigi{ 66262153Sluigi /* TODO XXX please use the FreeBSD implementation for this. */ 67262153Sluigi uint16_t *words = (uint16_t *)data; 68262153Sluigi int nw = len / 2; 69262153Sluigi int i; 70262153Sluigi 71262153Sluigi for (i = 0; i < nw; i++) 72262153Sluigi cur_sum += be16toh(words[i]); 73262153Sluigi 74262153Sluigi if (len & 1) 75262153Sluigi cur_sum += (data[len-1] << 8); 76262153Sluigi 77262153Sluigi return cur_sum; 78262153Sluigi} 79262153Sluigi 80262153Sluigi/* Fold a raw checksum: 'cur_sum' is in host byte order, while the 81262153Sluigi * return value is in network byte order. 82262153Sluigi */ 83262153Sluigiuint16_t nm_csum_fold(rawsum_t cur_sum) 84262153Sluigi{ 85262153Sluigi /* TODO XXX please use the FreeBSD implementation for this. */ 86262153Sluigi while (cur_sum >> 16) 87262153Sluigi cur_sum = (cur_sum & 0xFFFF) + (cur_sum >> 16); 88262153Sluigi 89262153Sluigi return htobe16((~cur_sum) & 0xFFFF); 90262153Sluigi} 91262153Sluigi 92262153Sluigiuint16_t nm_csum_ipv4(struct nm_iphdr *iph) 93262153Sluigi{ 94262153Sluigi#if 0 95262153Sluigi return in_cksum_hdr((void *)iph); 96262153Sluigi#else 97262153Sluigi return nm_csum_fold(nm_csum_raw((uint8_t*)iph, sizeof(struct nm_iphdr), 0)); 98262153Sluigi#endif 99262153Sluigi} 100262153Sluigi 101262153Sluigivoid nm_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, 102262153Sluigi size_t datalen, uint16_t *check) 103262153Sluigi{ 104262237Sluigi#ifdef INET 105262153Sluigi uint16_t pseudolen = datalen + iph->protocol; 106262153Sluigi 107262153Sluigi /* Compute and insert the pseudo-header cheksum. */ 108262153Sluigi *check = in_pseudo(iph->saddr, iph->daddr, 109262153Sluigi htobe16(pseudolen)); 110262153Sluigi /* Compute the checksum on TCP/UDP header + payload 111262153Sluigi * (includes the pseudo-header). 112262153Sluigi */ 113262153Sluigi *check = nm_csum_fold(nm_csum_raw(data, datalen, 0)); 114262237Sluigi#else 115262237Sluigi static int notsupported = 0; 116262237Sluigi if (!notsupported) { 117262237Sluigi notsupported = 1; 118262237Sluigi D("inet4 segmentation not supported"); 119262237Sluigi } 120262237Sluigi#endif 121262153Sluigi} 122262153Sluigi 123262153Sluigivoid nm_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, 124262153Sluigi size_t datalen, uint16_t *check) 125262153Sluigi{ 126262153Sluigi#ifdef INET6 127262153Sluigi *check = in6_cksum_pseudo((void*)ip6h, datalen, ip6h->nexthdr, 0); 128262153Sluigi *check = nm_csum_fold(nm_csum_raw(data, datalen, 0)); 129262153Sluigi#else 130262153Sluigi static int notsupported = 0; 131262153Sluigi if (!notsupported) { 132262153Sluigi notsupported = 1; 133262153Sluigi D("inet6 segmentation not supported"); 134262153Sluigi } 135262153Sluigi#endif 136262153Sluigi} 137262153Sluigi 138262153Sluigi 139262153Sluigi/* 140262153Sluigi * Intercept the rx routine in the standard device driver. 141262153Sluigi * Second argument is non-zero to intercept, 0 to restore 142262153Sluigi */ 143262153Sluigiint 144262153Sluiginetmap_catch_rx(struct netmap_adapter *na, int intercept) 145262153Sluigi{ 146262153Sluigi struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 147262153Sluigi struct ifnet *ifp = na->ifp; 148262153Sluigi 149262153Sluigi if (intercept) { 150262153Sluigi if (gna->save_if_input) { 151262153Sluigi D("cannot intercept again"); 152262153Sluigi return EINVAL; /* already set */ 153262153Sluigi } 154262153Sluigi gna->save_if_input = ifp->if_input; 155262153Sluigi ifp->if_input = generic_rx_handler; 156262153Sluigi } else { 157262153Sluigi if (!gna->save_if_input){ 158262153Sluigi D("cannot restore"); 159262153Sluigi return EINVAL; /* not saved */ 160262153Sluigi } 161262153Sluigi ifp->if_input = gna->save_if_input; 162262153Sluigi gna->save_if_input = NULL; 163262153Sluigi } 164262153Sluigi 165262153Sluigi return 0; 166262153Sluigi} 167262153Sluigi 168262153Sluigi 169262153Sluigi/* 170262153Sluigi * Intercept the packet steering routine in the tx path, 171262153Sluigi * so that we can decide which queue is used for an mbuf. 172262153Sluigi * Second argument is non-zero to intercept, 0 to restore. 173262153Sluigi * On freebsd we just intercept if_transmit. 174262153Sluigi */ 175262153Sluigivoid 176262153Sluiginetmap_catch_tx(struct netmap_generic_adapter *gna, int enable) 177262153Sluigi{ 178262153Sluigi struct netmap_adapter *na = &gna->up.up; 179262153Sluigi struct ifnet *ifp = na->ifp; 180262153Sluigi 181262153Sluigi if (enable) { 182262153Sluigi na->if_transmit = ifp->if_transmit; 183262153Sluigi ifp->if_transmit = netmap_transmit; 184262153Sluigi } else { 185262153Sluigi ifp->if_transmit = na->if_transmit; 186262153Sluigi } 187262153Sluigi} 188262153Sluigi 189262153Sluigi 190262153Sluigi/* 191262153Sluigi * Transmit routine used by generic_netmap_txsync(). Returns 0 on success 192262153Sluigi * and non-zero on error (which may be packet drops or other errors). 193262153Sluigi * addr and len identify the netmap buffer, m is the (preallocated) 194262153Sluigi * mbuf to use for transmissions. 195262153Sluigi * 196262153Sluigi * We should add a reference to the mbuf so the m_freem() at the end 197262153Sluigi * of the transmission does not consume resources. 198262153Sluigi * 199262153Sluigi * On FreeBSD, and on multiqueue cards, we can force the queue using 200262153Sluigi * if ((m->m_flags & M_FLOWID) != 0) 201262153Sluigi * i = m->m_pkthdr.flowid % adapter->num_queues; 202262153Sluigi * else 203262153Sluigi * i = curcpu % adapter->num_queues; 204262153Sluigi * 205262153Sluigi */ 206262153Sluigiint 207262153Sluigigeneric_xmit_frame(struct ifnet *ifp, struct mbuf *m, 208262153Sluigi void *addr, u_int len, u_int ring_nr) 209262153Sluigi{ 210262153Sluigi int ret; 211262153Sluigi 212262153Sluigi m->m_len = m->m_pkthdr.len = 0; 213262153Sluigi 214262153Sluigi // copy data to the mbuf 215262153Sluigi m_copyback(m, 0, len, addr); 216262153Sluigi // inc refcount. We are alone, so we can skip the atomic 217262153Sluigi atomic_fetchadd_int(m->m_ext.ref_cnt, 1); 218262153Sluigi m->m_flags |= M_FLOWID; 219262153Sluigi m->m_pkthdr.flowid = ring_nr; 220262153Sluigi m->m_pkthdr.rcvif = ifp; /* used for tx notification */ 221262153Sluigi ret = NA(ifp)->if_transmit(ifp, m); 222262153Sluigi return ret; 223262153Sluigi} 224262153Sluigi 225262153Sluigi 226262153Sluigi/* 227262153Sluigi * The following two functions are empty until we have a generic 228262153Sluigi * way to extract the info from the ifp 229262153Sluigi */ 230262153Sluigiint 231262153Sluigigeneric_find_num_desc(struct ifnet *ifp, unsigned int *tx, unsigned int *rx) 232262153Sluigi{ 233262153Sluigi D("called"); 234262153Sluigi return 0; 235262153Sluigi} 236262153Sluigi 237262153Sluigi 238262153Sluigivoid 239262153Sluigigeneric_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq) 240262153Sluigi{ 241262153Sluigi D("called"); 242262153Sluigi *txq = netmap_generic_rings; 243262153Sluigi *rxq = netmap_generic_rings; 244262153Sluigi} 245262153Sluigi 246262153Sluigi 247262153Sluigivoid netmap_mitigation_init(struct nm_generic_mit *mit, struct netmap_adapter *na) 248262153Sluigi{ 249262153Sluigi ND("called"); 250262153Sluigi mit->mit_pending = 0; 251262153Sluigi mit->mit_na = na; 252262153Sluigi} 253262153Sluigi 254262153Sluigi 255262153Sluigivoid netmap_mitigation_start(struct nm_generic_mit *mit) 256262153Sluigi{ 257262153Sluigi ND("called"); 258262153Sluigi} 259262153Sluigi 260262153Sluigi 261262153Sluigivoid netmap_mitigation_restart(struct nm_generic_mit *mit) 262262153Sluigi{ 263262153Sluigi ND("called"); 264262153Sluigi} 265262153Sluigi 266262153Sluigi 267262153Sluigiint netmap_mitigation_active(struct nm_generic_mit *mit) 268262153Sluigi{ 269262153Sluigi ND("called"); 270262153Sluigi return 0; 271262153Sluigi} 272262153Sluigi 273262153Sluigi 274262153Sluigivoid netmap_mitigation_cleanup(struct nm_generic_mit *mit) 275262153Sluigi{ 276262153Sluigi ND("called"); 277262153Sluigi} 278262153Sluigi 279262153Sluigi 280262153Sluigi/* 281262153Sluigi * In order to track whether pages are still mapped, we hook into 282262153Sluigi * the standard cdev_pager and intercept the constructor and 283262153Sluigi * destructor. 284262153Sluigi */ 285262153Sluigi 286262153Sluigistruct netmap_vm_handle_t { 287262153Sluigi struct cdev *dev; 288262153Sluigi struct netmap_priv_d *priv; 289262153Sluigi}; 290262153Sluigi 291262153Sluigi 292262153Sluigistatic int 293262153Sluiginetmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 294262153Sluigi vm_ooffset_t foff, struct ucred *cred, u_short *color) 295262153Sluigi{ 296262153Sluigi struct netmap_vm_handle_t *vmh = handle; 297262153Sluigi 298262153Sluigi if (netmap_verbose) 299262153Sluigi D("handle %p size %jd prot %d foff %jd", 300262153Sluigi handle, (intmax_t)size, prot, (intmax_t)foff); 301262153Sluigi dev_ref(vmh->dev); 302262153Sluigi return 0; 303262153Sluigi} 304262153Sluigi 305262153Sluigi 306262153Sluigistatic void 307262153Sluiginetmap_dev_pager_dtor(void *handle) 308262153Sluigi{ 309262153Sluigi struct netmap_vm_handle_t *vmh = handle; 310262153Sluigi struct cdev *dev = vmh->dev; 311262153Sluigi struct netmap_priv_d *priv = vmh->priv; 312262153Sluigi 313262153Sluigi if (netmap_verbose) 314262153Sluigi D("handle %p", handle); 315262153Sluigi netmap_dtor(priv); 316262153Sluigi free(vmh, M_DEVBUF); 317262153Sluigi dev_rel(dev); 318262153Sluigi} 319262153Sluigi 320262153Sluigi 321262153Sluigistatic int 322262153Sluiginetmap_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, 323262153Sluigi int prot, vm_page_t *mres) 324262153Sluigi{ 325262153Sluigi struct netmap_vm_handle_t *vmh = object->handle; 326262153Sluigi struct netmap_priv_d *priv = vmh->priv; 327262153Sluigi vm_paddr_t paddr; 328262153Sluigi vm_page_t page; 329262153Sluigi vm_memattr_t memattr; 330262153Sluigi vm_pindex_t pidx; 331262153Sluigi 332262153Sluigi ND("object %p offset %jd prot %d mres %p", 333262153Sluigi object, (intmax_t)offset, prot, mres); 334262153Sluigi memattr = object->memattr; 335262153Sluigi pidx = OFF_TO_IDX(offset); 336262153Sluigi paddr = netmap_mem_ofstophys(priv->np_mref, offset); 337262153Sluigi if (paddr == 0) 338262153Sluigi return VM_PAGER_FAIL; 339262153Sluigi 340262153Sluigi if (((*mres)->flags & PG_FICTITIOUS) != 0) { 341262153Sluigi /* 342262153Sluigi * If the passed in result page is a fake page, update it with 343262153Sluigi * the new physical address. 344262153Sluigi */ 345262153Sluigi page = *mres; 346262153Sluigi vm_page_updatefake(page, paddr, memattr); 347262153Sluigi } else { 348262153Sluigi /* 349262153Sluigi * Replace the passed in reqpage page with our own fake page and 350262153Sluigi * free up the all of the original pages. 351262153Sluigi */ 352262153Sluigi#ifndef VM_OBJECT_WUNLOCK /* FreeBSD < 10.x */ 353262153Sluigi#define VM_OBJECT_WUNLOCK VM_OBJECT_UNLOCK 354262153Sluigi#define VM_OBJECT_WLOCK VM_OBJECT_LOCK 355262153Sluigi#endif /* VM_OBJECT_WUNLOCK */ 356262153Sluigi 357262153Sluigi VM_OBJECT_WUNLOCK(object); 358262153Sluigi page = vm_page_getfake(paddr, memattr); 359262153Sluigi VM_OBJECT_WLOCK(object); 360262153Sluigi vm_page_lock(*mres); 361262153Sluigi vm_page_free(*mres); 362262153Sluigi vm_page_unlock(*mres); 363262153Sluigi *mres = page; 364262153Sluigi vm_page_insert(page, object, pidx); 365262153Sluigi } 366262153Sluigi page->valid = VM_PAGE_BITS_ALL; 367262153Sluigi return (VM_PAGER_OK); 368262153Sluigi} 369262153Sluigi 370262153Sluigi 371262153Sluigistatic struct cdev_pager_ops netmap_cdev_pager_ops = { 372262153Sluigi .cdev_pg_ctor = netmap_dev_pager_ctor, 373262153Sluigi .cdev_pg_dtor = netmap_dev_pager_dtor, 374262153Sluigi .cdev_pg_fault = netmap_dev_pager_fault, 375262153Sluigi}; 376262153Sluigi 377262153Sluigi 378262153Sluigistatic int 379262153Sluiginetmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff, 380262153Sluigi vm_size_t objsize, vm_object_t *objp, int prot) 381262153Sluigi{ 382262153Sluigi int error; 383262153Sluigi struct netmap_vm_handle_t *vmh; 384262153Sluigi struct netmap_priv_d *priv; 385262153Sluigi vm_object_t obj; 386262153Sluigi 387262153Sluigi if (netmap_verbose) 388262153Sluigi D("cdev %p foff %jd size %jd objp %p prot %d", cdev, 389262153Sluigi (intmax_t )*foff, (intmax_t )objsize, objp, prot); 390262153Sluigi 391262153Sluigi vmh = malloc(sizeof(struct netmap_vm_handle_t), M_DEVBUF, 392262153Sluigi M_NOWAIT | M_ZERO); 393262153Sluigi if (vmh == NULL) 394262153Sluigi return ENOMEM; 395262153Sluigi vmh->dev = cdev; 396262153Sluigi 397262153Sluigi NMG_LOCK(); 398262153Sluigi error = devfs_get_cdevpriv((void**)&priv); 399262153Sluigi if (error) 400262153Sluigi goto err_unlock; 401262153Sluigi vmh->priv = priv; 402262153Sluigi priv->np_refcount++; 403262153Sluigi NMG_UNLOCK(); 404262153Sluigi 405262153Sluigi error = netmap_get_memory(priv); 406262153Sluigi if (error) 407262153Sluigi goto err_deref; 408262153Sluigi 409262153Sluigi obj = cdev_pager_allocate(vmh, OBJT_DEVICE, 410262153Sluigi &netmap_cdev_pager_ops, objsize, prot, 411262153Sluigi *foff, NULL); 412262153Sluigi if (obj == NULL) { 413262153Sluigi D("cdev_pager_allocate failed"); 414262153Sluigi error = EINVAL; 415262153Sluigi goto err_deref; 416262153Sluigi } 417262153Sluigi 418262153Sluigi *objp = obj; 419262153Sluigi return 0; 420262153Sluigi 421262153Sluigierr_deref: 422262153Sluigi NMG_LOCK(); 423262153Sluigi priv->np_refcount--; 424262153Sluigierr_unlock: 425262153Sluigi NMG_UNLOCK(); 426262153Sluigi// err: 427262153Sluigi free(vmh, M_DEVBUF); 428262153Sluigi return error; 429262153Sluigi} 430262153Sluigi 431262153Sluigi 432262153Sluigi// XXX can we remove this ? 433262153Sluigistatic int 434262153Sluiginetmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 435262153Sluigi{ 436262153Sluigi if (netmap_verbose) 437262153Sluigi D("dev %p fflag 0x%x devtype %d td %p", 438262153Sluigi dev, fflag, devtype, td); 439262153Sluigi return 0; 440262153Sluigi} 441262153Sluigi 442262153Sluigi 443262153Sluigistatic int 444262153Sluiginetmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 445262153Sluigi{ 446262153Sluigi struct netmap_priv_d *priv; 447262153Sluigi int error; 448262153Sluigi 449262153Sluigi (void)dev; 450262153Sluigi (void)oflags; 451262153Sluigi (void)devtype; 452262153Sluigi (void)td; 453262153Sluigi 454262153Sluigi // XXX wait or nowait ? 455262153Sluigi priv = malloc(sizeof(struct netmap_priv_d), M_DEVBUF, 456262153Sluigi M_NOWAIT | M_ZERO); 457262153Sluigi if (priv == NULL) 458262153Sluigi return ENOMEM; 459262153Sluigi 460262153Sluigi error = devfs_set_cdevpriv(priv, netmap_dtor); 461262153Sluigi if (error) 462262153Sluigi return error; 463262153Sluigi 464262153Sluigi priv->np_refcount = 1; 465262153Sluigi 466262153Sluigi return 0; 467262153Sluigi} 468262153Sluigi 469262153Sluigi/******************** kqueue support ****************/ 470262153Sluigi 471262153Sluigi/* 472262153Sluigi * The OS_selwakeup also needs to issue a KNOTE_UNLOCKED. 473262153Sluigi * We use a non-zero argument to distinguish the call from the one 474262153Sluigi * in kevent_scan() which instead also needs to run netmap_poll(). 475262153Sluigi * The knote uses a global mutex for the time being. We might 476262153Sluigi * try to reuse the one in the si, but it is not allocated 477262153Sluigi * permanently so it might be a bit tricky. 478262153Sluigi * 479262153Sluigi * The *kqfilter function registers one or another f_event 480262153Sluigi * depending on read or write mode. 481262153Sluigi * In the call to f_event() td_fpop is NULL so any child function 482262153Sluigi * calling devfs_get_cdevpriv() would fail - and we need it in 483262153Sluigi * netmap_poll(). As a workaround we store priv into kn->kn_hook 484262153Sluigi * and pass it as first argument to netmap_poll(), which then 485262153Sluigi * uses the failure to tell that we are called from f_event() 486262153Sluigi * and do not need the selrecord(). 487262153Sluigi */ 488262153Sluigi 489262153Sluigivoid freebsd_selwakeup(struct selinfo *si, int pri); 490262153Sluigi 491262153Sluigivoid 492262153Sluigifreebsd_selwakeup(struct selinfo *si, int pri) 493262153Sluigi{ 494262153Sluigi if (netmap_verbose) 495262153Sluigi D("on knote %p", &si->si_note); 496262153Sluigi selwakeuppri(si, pri); 497262153Sluigi /* use a non-zero hint to tell the notification from the 498262153Sluigi * call done in kqueue_scan() which uses 0 499262153Sluigi */ 500262153Sluigi KNOTE_UNLOCKED(&si->si_note, 0x100 /* notification */); 501262153Sluigi} 502262153Sluigi 503262153Sluigistatic void 504262153Sluiginetmap_knrdetach(struct knote *kn) 505262153Sluigi{ 506262153Sluigi struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; 507262153Sluigi struct selinfo *si = priv->np_rxsi; 508262153Sluigi 509262153Sluigi D("remove selinfo %p", si); 510262153Sluigi knlist_remove(&si->si_note, kn, 0); 511262153Sluigi} 512262153Sluigi 513262153Sluigistatic void 514262153Sluiginetmap_knwdetach(struct knote *kn) 515262153Sluigi{ 516262153Sluigi struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; 517262153Sluigi struct selinfo *si = priv->np_txsi; 518262153Sluigi 519262153Sluigi D("remove selinfo %p", si); 520262153Sluigi knlist_remove(&si->si_note, kn, 0); 521262153Sluigi} 522262153Sluigi 523262153Sluigi/* 524262153Sluigi * callback from notifies (generated externally) and our 525262153Sluigi * calls to kevent(). The former we just return 1 (ready) 526262153Sluigi * since we do not know better. 527262153Sluigi * In the latter we call netmap_poll and return 0/1 accordingly. 528262153Sluigi */ 529262153Sluigistatic int 530262153Sluiginetmap_knrw(struct knote *kn, long hint, int events) 531262153Sluigi{ 532262153Sluigi struct netmap_priv_d *priv; 533262153Sluigi int revents; 534262153Sluigi 535262153Sluigi if (hint != 0) { 536262153Sluigi ND(5, "call from notify"); 537262153Sluigi return 1; /* assume we are ready */ 538262153Sluigi } 539262153Sluigi priv = kn->kn_hook; 540262153Sluigi /* the notification may come from an external thread, 541262153Sluigi * in which case we do not want to run the netmap_poll 542262153Sluigi * This should be filtered above, but check just in case. 543262153Sluigi */ 544262153Sluigi if (curthread != priv->np_td) { /* should not happen */ 545262153Sluigi RD(5, "curthread changed %p %p", curthread, priv->np_td); 546262153Sluigi return 1; 547262153Sluigi } else { 548262153Sluigi revents = netmap_poll((void *)priv, events, curthread); 549262153Sluigi return (events & revents) ? 1 : 0; 550262153Sluigi } 551262153Sluigi} 552262153Sluigi 553262153Sluigistatic int 554262153Sluiginetmap_knread(struct knote *kn, long hint) 555262153Sluigi{ 556262153Sluigi return netmap_knrw(kn, hint, POLLIN); 557262153Sluigi} 558262153Sluigi 559262153Sluigistatic int 560262153Sluiginetmap_knwrite(struct knote *kn, long hint) 561262153Sluigi{ 562262153Sluigi return netmap_knrw(kn, hint, POLLOUT); 563262153Sluigi} 564262153Sluigi 565262153Sluigistatic struct filterops netmap_rfiltops = { 566262153Sluigi .f_isfd = 1, 567262153Sluigi .f_detach = netmap_knrdetach, 568262153Sluigi .f_event = netmap_knread, 569262153Sluigi}; 570262153Sluigi 571262153Sluigistatic struct filterops netmap_wfiltops = { 572262153Sluigi .f_isfd = 1, 573262153Sluigi .f_detach = netmap_knwdetach, 574262153Sluigi .f_event = netmap_knwrite, 575262153Sluigi}; 576262153Sluigi 577262153Sluigi 578262153Sluigi/* 579262153Sluigi * This is called when a thread invokes kevent() to record 580262153Sluigi * a change in the configuration of the kqueue(). 581262153Sluigi * The 'priv' should be the same as in the netmap device. 582262153Sluigi */ 583262153Sluigistatic int 584262153Sluiginetmap_kqfilter(struct cdev *dev, struct knote *kn) 585262153Sluigi{ 586262153Sluigi struct netmap_priv_d *priv; 587262153Sluigi int error; 588262153Sluigi struct netmap_adapter *na; 589262153Sluigi struct selinfo *si; 590262153Sluigi int ev = kn->kn_filter; 591262153Sluigi 592262153Sluigi if (ev != EVFILT_READ && ev != EVFILT_WRITE) { 593262153Sluigi D("bad filter request %d", ev); 594262153Sluigi return 1; 595262153Sluigi } 596262153Sluigi error = devfs_get_cdevpriv((void**)&priv); 597262153Sluigi if (error) { 598262153Sluigi D("device not yet setup"); 599262153Sluigi return 1; 600262153Sluigi } 601262153Sluigi na = priv->np_na; 602262153Sluigi if (na == NULL) { 603262153Sluigi D("no netmap adapter for this file descriptor"); 604262153Sluigi return 1; 605262153Sluigi } 606262153Sluigi /* the si is indicated in the priv */ 607262153Sluigi si = (ev == EVFILT_WRITE) ? priv->np_txsi : priv->np_rxsi; 608262153Sluigi // XXX lock(priv) ? 609262153Sluigi kn->kn_fop = (ev == EVFILT_WRITE) ? 610262153Sluigi &netmap_wfiltops : &netmap_rfiltops; 611262153Sluigi kn->kn_hook = priv; 612262153Sluigi knlist_add(&si->si_note, kn, 1); 613262153Sluigi // XXX unlock(priv) 614262153Sluigi ND("register %p %s td %p priv %p kn %p np_nifp %p kn_fp/fpop %s", 615262153Sluigi na, na->ifp->if_xname, curthread, priv, kn, 616262153Sluigi priv->np_nifp, 617262153Sluigi kn->kn_fp == curthread->td_fpop ? "match" : "MISMATCH"); 618262153Sluigi return 0; 619262153Sluigi} 620262153Sluigi 621262153Sluigistruct cdevsw netmap_cdevsw = { 622262153Sluigi .d_version = D_VERSION, 623262153Sluigi .d_name = "netmap", 624262153Sluigi .d_open = netmap_open, 625262153Sluigi .d_mmap_single = netmap_mmap_single, 626262153Sluigi .d_ioctl = netmap_ioctl, 627262153Sluigi .d_poll = netmap_poll, 628262153Sluigi .d_kqfilter = netmap_kqfilter, 629262153Sluigi .d_close = netmap_close, 630262153Sluigi}; 631262153Sluigi/*--- end of kqueue support ----*/ 632262153Sluigi 633262153Sluigi/* 634262153Sluigi * Kernel entry point. 635262153Sluigi * 636262153Sluigi * Initialize/finalize the module and return. 637262153Sluigi * 638262153Sluigi * Return 0 on success, errno on failure. 639262153Sluigi */ 640262153Sluigistatic int 641262153Sluiginetmap_loader(__unused struct module *module, int event, __unused void *arg) 642262153Sluigi{ 643262153Sluigi int error = 0; 644262153Sluigi 645262153Sluigi switch (event) { 646262153Sluigi case MOD_LOAD: 647262153Sluigi error = netmap_init(); 648262153Sluigi break; 649262153Sluigi 650262153Sluigi case MOD_UNLOAD: 651262153Sluigi netmap_fini(); 652262153Sluigi break; 653262153Sluigi 654262153Sluigi default: 655262153Sluigi error = EOPNOTSUPP; 656262153Sluigi break; 657262153Sluigi } 658262153Sluigi 659262153Sluigi return (error); 660262153Sluigi} 661262153Sluigi 662262153Sluigi 663262153SluigiDEV_MODULE(netmap, netmap_loader, NULL); 664