1331722Seadler/* 2260368Sluigi * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. 3259412Sluigi * 4259412Sluigi * Redistribution and use in source and binary forms, with or without 5259412Sluigi * modification, are permitted provided that the following conditions 6259412Sluigi * are met: 7259412Sluigi * 1. Redistributions of source code must retain the above copyright 8259412Sluigi * notice, this list of conditions and the following disclaimer. 9259412Sluigi * 2. Redistributions in binary form must reproduce the above copyright 10259412Sluigi * notice, this list of conditions and the following disclaimer in the 11259412Sluigi * documentation and/or other materials provided with the distribution. 12259412Sluigi * 13259412Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14259412Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15259412Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16259412Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17259412Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18259412Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19259412Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20259412Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21259412Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22259412Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23259412Sluigi * SUCH DAMAGE. 24259412Sluigi */ 25259412Sluigi 26259412Sluigi/* $FreeBSD: stable/11/sys/dev/netmap/netmap_freebsd.c 369514 2021-03-23 21:21:46Z git2svn $ */ 27285349Sluigi#include "opt_inet.h" 28285349Sluigi#include "opt_inet6.h" 29259412Sluigi 30341477Svmaffione#include <sys/param.h> 31259412Sluigi#include <sys/module.h> 32259412Sluigi#include <sys/errno.h> 33341477Svmaffione#include <sys/jail.h> 34261909Sluigi#include <sys/poll.h> /* POLLIN, POLLOUT */ 35259412Sluigi#include <sys/kernel.h> /* types used in module initialization */ 36341477Svmaffione#include <sys/conf.h> /* DEV_MODULE_ORDERED */ 37261909Sluigi#include <sys/endian.h> 38341477Svmaffione#include <sys/syscallsubr.h> /* kern_ioctl() */ 39259412Sluigi 40259412Sluigi#include <sys/rwlock.h> 41259412Sluigi 42259412Sluigi#include <vm/vm.h> /* vtophys */ 43259412Sluigi#include <vm/pmap.h> /* vtophys */ 44259412Sluigi#include <vm/vm_param.h> 45259412Sluigi#include <vm/vm_object.h> 46259412Sluigi#include <vm/vm_page.h> 47259412Sluigi#include <vm/vm_pager.h> 48259412Sluigi#include <vm/uma.h> 49259412Sluigi 50259412Sluigi 51259412Sluigi#include <sys/malloc.h> 52259412Sluigi#include <sys/socket.h> /* sockaddrs */ 53259412Sluigi#include <sys/selinfo.h> 54341477Svmaffione#include <sys/kthread.h> /* kthread_add() */ 55341477Svmaffione#include <sys/proc.h> /* PROC_LOCK() */ 56341477Svmaffione#include <sys/unistd.h> /* RFNOWAIT */ 57341477Svmaffione#include <sys/sched.h> /* sched_bind() */ 58341477Svmaffione#include <sys/smp.h> /* mp_maxid */ 59344509Svmaffione#include <sys/taskqueue.h> /* taskqueue_enqueue(), taskqueue_create(), ... */ 60259412Sluigi#include <net/if.h> 61259412Sluigi#include <net/if_var.h> 62270063Sluigi#include <net/if_types.h> /* IFT_ETHER */ 63270063Sluigi#include <net/ethernet.h> /* ether_ifdetach */ 64270063Sluigi#include <net/if_dl.h> /* LLADDR */ 65259412Sluigi#include <machine/bus.h> /* bus_dmamap_* */ 66261909Sluigi#include <netinet/in.h> /* in6_cksum_pseudo() */ 67261909Sluigi#include <machine/in_cksum.h> /* in_pseudo(), in_cksum_hdr() */ 68259412Sluigi 69259412Sluigi#include <net/netmap.h> 70259412Sluigi#include <dev/netmap/netmap_kern.h> 71341477Svmaffione#include <net/netmap_virt.h> 72259412Sluigi#include <dev/netmap/netmap_mem2.h> 73259412Sluigi 74259412Sluigi 75259412Sluigi/* ======================== FREEBSD-SPECIFIC ROUTINES ================== */ 76259412Sluigi 77344509Svmaffionestatic void 78344509Svmaffionenm_kqueue_notify(void *opaque, int pending) 79344509Svmaffione{ 80344509Svmaffione struct nm_selinfo *si = opaque; 81344509Svmaffione 82344509Svmaffione /* We use a non-zero hint to distinguish this notification call 83344509Svmaffione * from the call done in kqueue_scan(), which uses hint=0. 84344509Svmaffione */ 85344509Svmaffione KNOTE_UNLOCKED(&si->si.si_note, /*hint=*/0x100); 86341477Svmaffione} 87341477Svmaffione 88344509Svmaffioneint nm_os_selinfo_init(NM_SELINFO_T *si, const char *name) { 89344509Svmaffione int err; 90344509Svmaffione 91344509Svmaffione TASK_INIT(&si->ntfytask, 0, nm_kqueue_notify, si); 92344509Svmaffione si->ntfytq = taskqueue_create(name, M_NOWAIT, 93344509Svmaffione taskqueue_thread_enqueue, &si->ntfytq); 94344509Svmaffione if (si->ntfytq == NULL) 95344509Svmaffione return -ENOMEM; 96344509Svmaffione err = taskqueue_start_threads(&si->ntfytq, 1, PI_NET, "tq %s", name); 97344509Svmaffione if (err) { 98344509Svmaffione taskqueue_free(si->ntfytq); 99344509Svmaffione si->ntfytq = NULL; 100344509Svmaffione return err; 101344509Svmaffione } 102344509Svmaffione 103344509Svmaffione snprintf(si->mtxname, sizeof(si->mtxname), "nmkl%s", name); 104344509Svmaffione mtx_init(&si->m, si->mtxname, NULL, MTX_DEF); 105344509Svmaffione knlist_init_mtx(&si->si.si_note, &si->m); 106344509Svmaffione si->kqueue_users = 0; 107344509Svmaffione 108344509Svmaffione return (0); 109344509Svmaffione} 110344509Svmaffione 111341477Svmaffionevoid 112341477Svmaffionenm_os_selinfo_uninit(NM_SELINFO_T *si) 113341477Svmaffione{ 114344509Svmaffione if (si->ntfytq == NULL) { 115344509Svmaffione return; /* si was not initialized */ 116344509Svmaffione } 117344509Svmaffione taskqueue_drain(si->ntfytq, &si->ntfytask); 118344509Svmaffione taskqueue_free(si->ntfytq); 119344509Svmaffione si->ntfytq = NULL; 120343831Svmaffione knlist_delete(&si->si.si_note, curthread, /*islocked=*/0); 121341477Svmaffione knlist_destroy(&si->si.si_note); 122341477Svmaffione /* now we don't need the mutex anymore */ 123341477Svmaffione mtx_destroy(&si->m); 124341477Svmaffione} 125341477Svmaffione 126341477Svmaffionevoid * 127341477Svmaffionenm_os_malloc(size_t size) 128341477Svmaffione{ 129341477Svmaffione return malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 130341477Svmaffione} 131341477Svmaffione 132341477Svmaffionevoid * 133341477Svmaffionenm_os_realloc(void *addr, size_t new_size, size_t old_size __unused) 134341477Svmaffione{ 135341477Svmaffione return realloc(addr, new_size, M_DEVBUF, M_NOWAIT | M_ZERO); 136341477Svmaffione} 137341477Svmaffione 138341477Svmaffionevoid 139341477Svmaffionenm_os_free(void *addr) 140341477Svmaffione{ 141341477Svmaffione free(addr, M_DEVBUF); 142341477Svmaffione} 143341477Svmaffione 144341477Svmaffionevoid 145341477Svmaffionenm_os_ifnet_lock(void) 146341477Svmaffione{ 147341477Svmaffione IFNET_RLOCK(); 148341477Svmaffione} 149341477Svmaffione 150341477Svmaffionevoid 151341477Svmaffionenm_os_ifnet_unlock(void) 152341477Svmaffione{ 153341477Svmaffione IFNET_RUNLOCK(); 154341477Svmaffione} 155341477Svmaffione 156341477Svmaffionestatic int netmap_use_count = 0; 157341477Svmaffione 158341477Svmaffionevoid 159341477Svmaffionenm_os_get_module(void) 160341477Svmaffione{ 161341477Svmaffione netmap_use_count++; 162341477Svmaffione} 163341477Svmaffione 164341477Svmaffionevoid 165341477Svmaffionenm_os_put_module(void) 166341477Svmaffione{ 167341477Svmaffione netmap_use_count--; 168341477Svmaffione} 169341477Svmaffione 170341477Svmaffionestatic void 171341477Svmaffionenetmap_ifnet_arrival_handler(void *arg __unused, struct ifnet *ifp) 172341477Svmaffione{ 173341477Svmaffione netmap_undo_zombie(ifp); 174341477Svmaffione} 175341477Svmaffione 176341477Svmaffionestatic void 177341477Svmaffionenetmap_ifnet_departure_handler(void *arg __unused, struct ifnet *ifp) 178341477Svmaffione{ 179341477Svmaffione netmap_make_zombie(ifp); 180341477Svmaffione} 181341477Svmaffione 182341477Svmaffionestatic eventhandler_tag nm_ifnet_ah_tag; 183341477Svmaffionestatic eventhandler_tag nm_ifnet_dh_tag; 184341477Svmaffione 185341477Svmaffioneint 186341477Svmaffionenm_os_ifnet_init(void) 187341477Svmaffione{ 188341477Svmaffione nm_ifnet_ah_tag = 189341477Svmaffione EVENTHANDLER_REGISTER(ifnet_arrival_event, 190341477Svmaffione netmap_ifnet_arrival_handler, 191341477Svmaffione NULL, EVENTHANDLER_PRI_ANY); 192341477Svmaffione nm_ifnet_dh_tag = 193341477Svmaffione EVENTHANDLER_REGISTER(ifnet_departure_event, 194341477Svmaffione netmap_ifnet_departure_handler, 195341477Svmaffione NULL, EVENTHANDLER_PRI_ANY); 196341477Svmaffione return 0; 197341477Svmaffione} 198341477Svmaffione 199341477Svmaffionevoid 200341477Svmaffionenm_os_ifnet_fini(void) 201341477Svmaffione{ 202341477Svmaffione EVENTHANDLER_DEREGISTER(ifnet_arrival_event, 203341477Svmaffione nm_ifnet_ah_tag); 204341477Svmaffione EVENTHANDLER_DEREGISTER(ifnet_departure_event, 205341477Svmaffione nm_ifnet_dh_tag); 206341477Svmaffione} 207341477Svmaffione 208341477Svmaffioneunsigned 209341477Svmaffionenm_os_ifnet_mtu(struct ifnet *ifp) 210341477Svmaffione{ 211341477Svmaffione#if __FreeBSD_version < 1100030 212341477Svmaffione return ifp->if_data.ifi_mtu; 213341477Svmaffione#else /* __FreeBSD_version >= 1100030 */ 214341477Svmaffione return ifp->if_mtu; 215341477Svmaffione#endif 216341477Svmaffione} 217341477Svmaffione 218267180Sluigirawsum_t 219341477Svmaffionenm_os_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum) 220261909Sluigi{ 221261909Sluigi /* TODO XXX please use the FreeBSD implementation for this. */ 222261909Sluigi uint16_t *words = (uint16_t *)data; 223261909Sluigi int nw = len / 2; 224261909Sluigi int i; 225261909Sluigi 226261909Sluigi for (i = 0; i < nw; i++) 227261909Sluigi cur_sum += be16toh(words[i]); 228261909Sluigi 229261909Sluigi if (len & 1) 230261909Sluigi cur_sum += (data[len-1] << 8); 231261909Sluigi 232261909Sluigi return cur_sum; 233261909Sluigi} 234261909Sluigi 235261909Sluigi/* Fold a raw checksum: 'cur_sum' is in host byte order, while the 236261909Sluigi * return value is in network byte order. 237261909Sluigi */ 238267180Sluigiuint16_t 239341477Svmaffionenm_os_csum_fold(rawsum_t cur_sum) 240261909Sluigi{ 241261909Sluigi /* TODO XXX please use the FreeBSD implementation for this. */ 242261909Sluigi while (cur_sum >> 16) 243261909Sluigi cur_sum = (cur_sum & 0xFFFF) + (cur_sum >> 16); 244261909Sluigi 245261909Sluigi return htobe16((~cur_sum) & 0xFFFF); 246261909Sluigi} 247261909Sluigi 248341477Svmaffioneuint16_t nm_os_csum_ipv4(struct nm_iphdr *iph) 249261909Sluigi{ 250261909Sluigi#if 0 251261909Sluigi return in_cksum_hdr((void *)iph); 252261909Sluigi#else 253341477Svmaffione return nm_os_csum_fold(nm_os_csum_raw((uint8_t*)iph, sizeof(struct nm_iphdr), 0)); 254261909Sluigi#endif 255261909Sluigi} 256261909Sluigi 257267180Sluigivoid 258341477Svmaffionenm_os_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, 259261909Sluigi size_t datalen, uint16_t *check) 260261909Sluigi{ 261262238Sluigi#ifdef INET 262261909Sluigi uint16_t pseudolen = datalen + iph->protocol; 263261909Sluigi 264261909Sluigi /* Compute and insert the pseudo-header cheksum. */ 265261909Sluigi *check = in_pseudo(iph->saddr, iph->daddr, 266261909Sluigi htobe16(pseudolen)); 267261909Sluigi /* Compute the checksum on TCP/UDP header + payload 268261909Sluigi * (includes the pseudo-header). 269261909Sluigi */ 270341477Svmaffione *check = nm_os_csum_fold(nm_os_csum_raw(data, datalen, 0)); 271262238Sluigi#else 272262238Sluigi static int notsupported = 0; 273262238Sluigi if (!notsupported) { 274262238Sluigi notsupported = 1; 275343834Svmaffione nm_prerr("inet4 segmentation not supported"); 276262238Sluigi } 277262238Sluigi#endif 278261909Sluigi} 279261909Sluigi 280267180Sluigivoid 281341477Svmaffionenm_os_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, 282261909Sluigi size_t datalen, uint16_t *check) 283261909Sluigi{ 284261909Sluigi#ifdef INET6 285261909Sluigi *check = in6_cksum_pseudo((void*)ip6h, datalen, ip6h->nexthdr, 0); 286341477Svmaffione *check = nm_os_csum_fold(nm_os_csum_raw(data, datalen, 0)); 287261909Sluigi#else 288261909Sluigi static int notsupported = 0; 289261909Sluigi if (!notsupported) { 290261909Sluigi notsupported = 1; 291343834Svmaffione nm_prerr("inet6 segmentation not supported"); 292261909Sluigi } 293261909Sluigi#endif 294261909Sluigi} 295261909Sluigi 296341477Svmaffione/* on FreeBSD we send up one packet at a time */ 297341477Svmaffionevoid * 298341477Svmaffionenm_os_send_up(struct ifnet *ifp, struct mbuf *m, struct mbuf *prev) 299341477Svmaffione{ 300341477Svmaffione NA(ifp)->if_input(ifp, m); 301341477Svmaffione return NULL; 302341477Svmaffione} 303261909Sluigi 304341477Svmaffioneint 305341477Svmaffionenm_os_mbuf_has_csum_offld(struct mbuf *m) 306341477Svmaffione{ 307341477Svmaffione return m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_SCTP | 308341477Svmaffione CSUM_TCP_IPV6 | CSUM_UDP_IPV6 | 309341477Svmaffione CSUM_SCTP_IPV6); 310341477Svmaffione} 311341477Svmaffione 312341477Svmaffioneint 313341477Svmaffionenm_os_mbuf_has_seg_offld(struct mbuf *m) 314341477Svmaffione{ 315341477Svmaffione return m->m_pkthdr.csum_flags & CSUM_TSO; 316341477Svmaffione} 317341477Svmaffione 318341477Svmaffionestatic void 319341477Svmaffionefreebsd_generic_rx_handler(struct ifnet *ifp, struct mbuf *m) 320341477Svmaffione{ 321341477Svmaffione int stolen; 322341477Svmaffione 323343834Svmaffione if (unlikely(!NM_NA_VALID(ifp))) { 324343834Svmaffione nm_prlim(1, "Warning: RX packet intercepted, but no" 325343834Svmaffione " emulated adapter"); 326341477Svmaffione return; 327341477Svmaffione } 328341477Svmaffione 329341477Svmaffione stolen = generic_rx_handler(ifp, m); 330341477Svmaffione if (!stolen) { 331341477Svmaffione struct netmap_generic_adapter *gna = 332341477Svmaffione (struct netmap_generic_adapter *)NA(ifp); 333341477Svmaffione gna->save_if_input(ifp, m); 334341477Svmaffione } 335341477Svmaffione} 336341477Svmaffione 337259412Sluigi/* 338259412Sluigi * Intercept the rx routine in the standard device driver. 339259412Sluigi * Second argument is non-zero to intercept, 0 to restore 340259412Sluigi */ 341259412Sluigiint 342341477Svmaffionenm_os_catch_rx(struct netmap_generic_adapter *gna, int intercept) 343259412Sluigi{ 344285349Sluigi struct netmap_adapter *na = &gna->up.up; 345259412Sluigi struct ifnet *ifp = na->ifp; 346341477Svmaffione int ret = 0; 347259412Sluigi 348341477Svmaffione nm_os_ifnet_lock(); 349259412Sluigi if (intercept) { 350259412Sluigi if (gna->save_if_input) { 351343834Svmaffione nm_prerr("RX on %s already intercepted", na->name); 352343834Svmaffione ret = EBUSY; /* already set */ 353341477Svmaffione goto out; 354259412Sluigi } 355259412Sluigi gna->save_if_input = ifp->if_input; 356341477Svmaffione ifp->if_input = freebsd_generic_rx_handler; 357259412Sluigi } else { 358343834Svmaffione if (!gna->save_if_input) { 359343834Svmaffione nm_prerr("Failed to undo RX intercept on %s", 360343834Svmaffione na->name); 361341477Svmaffione ret = EINVAL; /* not saved */ 362341477Svmaffione goto out; 363259412Sluigi } 364259412Sluigi ifp->if_input = gna->save_if_input; 365259412Sluigi gna->save_if_input = NULL; 366259412Sluigi } 367341477Svmaffioneout: 368341477Svmaffione nm_os_ifnet_unlock(); 369259412Sluigi 370341477Svmaffione return ret; 371259412Sluigi} 372259412Sluigi 373260368Sluigi 374259412Sluigi/* 375259412Sluigi * Intercept the packet steering routine in the tx path, 376259412Sluigi * so that we can decide which queue is used for an mbuf. 377259412Sluigi * Second argument is non-zero to intercept, 0 to restore. 378261909Sluigi * On freebsd we just intercept if_transmit. 379259412Sluigi */ 380341477Svmaffioneint 381341477Svmaffionenm_os_catch_tx(struct netmap_generic_adapter *gna, int intercept) 382259412Sluigi{ 383260368Sluigi struct netmap_adapter *na = &gna->up.up; 384285349Sluigi struct ifnet *ifp = netmap_generic_getifp(gna); 385260368Sluigi 386341477Svmaffione nm_os_ifnet_lock(); 387341477Svmaffione if (intercept) { 388260368Sluigi na->if_transmit = ifp->if_transmit; 389260368Sluigi ifp->if_transmit = netmap_transmit; 390259412Sluigi } else { 391260368Sluigi ifp->if_transmit = na->if_transmit; 392259412Sluigi } 393341477Svmaffione nm_os_ifnet_unlock(); 394341477Svmaffione 395341477Svmaffione return 0; 396259412Sluigi} 397259412Sluigi 398260368Sluigi 399261909Sluigi/* 400261909Sluigi * Transmit routine used by generic_netmap_txsync(). Returns 0 on success 401259412Sluigi * and non-zero on error (which may be packet drops or other errors). 402259412Sluigi * addr and len identify the netmap buffer, m is the (preallocated) 403259412Sluigi * mbuf to use for transmissions. 404259412Sluigi * 405259412Sluigi * We should add a reference to the mbuf so the m_freem() at the end 406259412Sluigi * of the transmission does not consume resources. 407259412Sluigi * 408259412Sluigi * On FreeBSD, and on multiqueue cards, we can force the queue using 409275358Shselasky * if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 410259412Sluigi * i = m->m_pkthdr.flowid % adapter->num_queues; 411259412Sluigi * else 412259412Sluigi * i = curcpu % adapter->num_queues; 413259412Sluigi * 414259412Sluigi */ 415259412Sluigiint 416341477Svmaffionenm_os_generic_xmit_frame(struct nm_os_gen_arg *a) 417259412Sluigi{ 418259412Sluigi int ret; 419341477Svmaffione u_int len = a->len; 420341477Svmaffione struct ifnet *ifp = a->ifp; 421341477Svmaffione struct mbuf *m = a->m; 422259412Sluigi 423341477Svmaffione#if __FreeBSD_version < 1100000 424341477Svmaffione /* 425341477Svmaffione * Old FreeBSD versions. The mbuf has a cluster attached, 426341477Svmaffione * we need to copy from the cluster to the netmap buffer. 427341477Svmaffione */ 428341477Svmaffione if (MBUF_REFCNT(m) != 1) { 429343834Svmaffione nm_prerr("invalid refcnt %d for %p", MBUF_REFCNT(m), m); 430341477Svmaffione panic("in generic_xmit_frame"); 431341477Svmaffione } 432341477Svmaffione if (m->m_ext.ext_size < len) { 433343834Svmaffione nm_prlim(2, "size %d < len %d", m->m_ext.ext_size, len); 434341477Svmaffione len = m->m_ext.ext_size; 435341477Svmaffione } 436341477Svmaffione bcopy(a->addr, m->m_data, len); 437341477Svmaffione#else /* __FreeBSD_version >= 1100000 */ 438341477Svmaffione /* New FreeBSD versions. Link the external storage to 439341477Svmaffione * the netmap buffer, so that no copy is necessary. */ 440341477Svmaffione m->m_ext.ext_buf = m->m_data = a->addr; 441312783Sloos m->m_ext.ext_size = len; 442341477Svmaffione#endif /* __FreeBSD_version >= 1100000 */ 443259412Sluigi 444350010Svmaffione m->m_flags |= M_PKTHDR; 445267180Sluigi m->m_len = m->m_pkthdr.len = len; 446312783Sloos 447312783Sloos /* mbuf refcnt is not contended, no need to use atomic 448312783Sloos * (a memory barrier is enough). */ 449312783Sloos SET_MBUF_REFCNT(m, 2); 450275358Shselasky M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 451341477Svmaffione m->m_pkthdr.flowid = a->ring_nr; 452259412Sluigi m->m_pkthdr.rcvif = ifp; /* used for tx notification */ 453350010Svmaffione CURVNET_SET(ifp->if_vnet); 454260368Sluigi ret = NA(ifp)->if_transmit(ifp, m); 455350010Svmaffione CURVNET_RESTORE(); 456341477Svmaffione return ret ? -1 : 0; 457259412Sluigi} 458259412Sluigi 459260368Sluigi 460267170Sluigi#if __FreeBSD_version >= 1100005 461267170Sluigistruct netmap_adapter * 462267170Sluiginetmap_getna(if_t ifp) 463267170Sluigi{ 464267170Sluigi return (NA((struct ifnet *)ifp)); 465267170Sluigi} 466267170Sluigi#endif /* __FreeBSD_version >= 1100005 */ 467267170Sluigi 468259412Sluigi/* 469259412Sluigi * The following two functions are empty until we have a generic 470259412Sluigi * way to extract the info from the ifp 471259412Sluigi */ 472259412Sluigiint 473341477Svmaffionenm_os_generic_find_num_desc(struct ifnet *ifp, unsigned int *tx, unsigned int *rx) 474259412Sluigi{ 475259412Sluigi return 0; 476259412Sluigi} 477259412Sluigi 478260368Sluigi 479259412Sluigivoid 480341477Svmaffionenm_os_generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq) 481259412Sluigi{ 482341477Svmaffione unsigned num_rings = netmap_generic_rings ? netmap_generic_rings : 1; 483341477Svmaffione 484341477Svmaffione *txq = num_rings; 485341477Svmaffione *rxq = num_rings; 486259412Sluigi} 487259412Sluigi 488341477Svmaffionevoid 489341477Svmaffionenm_os_generic_set_features(struct netmap_generic_adapter *gna) 490341477Svmaffione{ 491260368Sluigi 492341477Svmaffione gna->rxsg = 1; /* Supported through m_copydata. */ 493341477Svmaffione gna->txqdisc = 0; /* Not supported. */ 494341477Svmaffione} 495341477Svmaffione 496267180Sluigivoid 497341477Svmaffionenm_os_mitigation_init(struct nm_generic_mit *mit, int idx, struct netmap_adapter *na) 498259412Sluigi{ 499261909Sluigi mit->mit_pending = 0; 500270063Sluigi mit->mit_ring_idx = idx; 501261909Sluigi mit->mit_na = na; 502259412Sluigi} 503259412Sluigi 504259412Sluigi 505267180Sluigivoid 506341477Svmaffionenm_os_mitigation_start(struct nm_generic_mit *mit) 507259412Sluigi{ 508259412Sluigi} 509259412Sluigi 510260368Sluigi 511267180Sluigivoid 512341477Svmaffionenm_os_mitigation_restart(struct nm_generic_mit *mit) 513259412Sluigi{ 514259412Sluigi} 515259412Sluigi 516260368Sluigi 517267180Sluigiint 518341477Svmaffionenm_os_mitigation_active(struct nm_generic_mit *mit) 519259412Sluigi{ 520343834Svmaffione 521259412Sluigi return 0; 522259412Sluigi} 523259412Sluigi 524260368Sluigi 525267180Sluigivoid 526341477Svmaffionenm_os_mitigation_cleanup(struct nm_generic_mit *mit) 527259412Sluigi{ 528259412Sluigi} 529259412Sluigi 530270063Sluigistatic int 531270063Sluiginm_vi_dummy(struct ifnet *ifp, u_long cmd, caddr_t addr) 532270063Sluigi{ 533343834Svmaffione 534270063Sluigi return EINVAL; 535270063Sluigi} 536260368Sluigi 537270063Sluigistatic void 538270063Sluiginm_vi_start(struct ifnet *ifp) 539270063Sluigi{ 540270063Sluigi panic("nm_vi_start() must not be called"); 541270063Sluigi} 542270063Sluigi 543259412Sluigi/* 544270063Sluigi * Index manager of persistent virtual interfaces. 545270063Sluigi * It is used to decide the lowest byte of the MAC address. 546270063Sluigi * We use the same algorithm with management of bridge port index. 547270063Sluigi */ 548270063Sluigi#define NM_VI_MAX 255 549270063Sluigistatic struct { 550270063Sluigi uint8_t index[NM_VI_MAX]; /* XXX just for a reasonable number */ 551270063Sluigi uint8_t active; 552270063Sluigi struct mtx lock; 553270063Sluigi} nm_vi_indices; 554270063Sluigi 555270063Sluigivoid 556341477Svmaffionenm_os_vi_init_index(void) 557270063Sluigi{ 558270063Sluigi int i; 559270063Sluigi for (i = 0; i < NM_VI_MAX; i++) 560270063Sluigi nm_vi_indices.index[i] = i; 561270063Sluigi nm_vi_indices.active = 0; 562270063Sluigi mtx_init(&nm_vi_indices.lock, "nm_vi_indices_lock", NULL, MTX_DEF); 563270063Sluigi} 564270063Sluigi 565270063Sluigi/* return -1 if no index available */ 566270063Sluigistatic int 567270063Sluiginm_vi_get_index(void) 568270063Sluigi{ 569270063Sluigi int ret; 570270063Sluigi 571270063Sluigi mtx_lock(&nm_vi_indices.lock); 572270063Sluigi ret = nm_vi_indices.active == NM_VI_MAX ? -1 : 573270063Sluigi nm_vi_indices.index[nm_vi_indices.active++]; 574270063Sluigi mtx_unlock(&nm_vi_indices.lock); 575270063Sluigi return ret; 576270063Sluigi} 577270063Sluigi 578270063Sluigistatic void 579270063Sluiginm_vi_free_index(uint8_t val) 580270063Sluigi{ 581270063Sluigi int i, lim; 582270063Sluigi 583270063Sluigi mtx_lock(&nm_vi_indices.lock); 584270063Sluigi lim = nm_vi_indices.active; 585270063Sluigi for (i = 0; i < lim; i++) { 586270063Sluigi if (nm_vi_indices.index[i] == val) { 587270063Sluigi /* swap index[lim-1] and j */ 588270063Sluigi int tmp = nm_vi_indices.index[lim-1]; 589270063Sluigi nm_vi_indices.index[lim-1] = val; 590270063Sluigi nm_vi_indices.index[i] = tmp; 591270063Sluigi nm_vi_indices.active--; 592270063Sluigi break; 593270063Sluigi } 594270063Sluigi } 595270063Sluigi if (lim == nm_vi_indices.active) 596343834Svmaffione nm_prerr("Index %u not found", val); 597270063Sluigi mtx_unlock(&nm_vi_indices.lock); 598270063Sluigi} 599270063Sluigi#undef NM_VI_MAX 600270063Sluigi 601270063Sluigi/* 602270063Sluigi * Implementation of a netmap-capable virtual interface that 603270063Sluigi * registered to the system. 604270063Sluigi * It is based on if_tap.c and ip_fw_log.c in FreeBSD 9. 605270063Sluigi * 606270063Sluigi * Note: Linux sets refcount to 0 on allocation of net_device, 607270063Sluigi * then increments it on registration to the system. 608270063Sluigi * FreeBSD sets refcount to 1 on if_alloc(), and does not 609270063Sluigi * increment this refcount on if_attach(). 610270063Sluigi */ 611270063Sluigiint 612341477Svmaffionenm_os_vi_persist(const char *name, struct ifnet **ret) 613270063Sluigi{ 614270063Sluigi struct ifnet *ifp; 615270063Sluigi u_short macaddr_hi; 616270063Sluigi uint32_t macaddr_mid; 617270063Sluigi u_char eaddr[6]; 618270063Sluigi int unit = nm_vi_get_index(); /* just to decide MAC address */ 619270063Sluigi 620270063Sluigi if (unit < 0) 621270063Sluigi return EBUSY; 622270063Sluigi /* 623270063Sluigi * We use the same MAC address generation method with tap 624270063Sluigi * except for the highest octet is 00:be instead of 00:bd 625270063Sluigi */ 626270063Sluigi macaddr_hi = htons(0x00be); /* XXX tap + 1 */ 627270063Sluigi macaddr_mid = (uint32_t) ticks; 628270063Sluigi bcopy(&macaddr_hi, eaddr, sizeof(short)); 629270063Sluigi bcopy(&macaddr_mid, &eaddr[2], sizeof(uint32_t)); 630270063Sluigi eaddr[5] = (uint8_t)unit; 631270063Sluigi 632270063Sluigi ifp = if_alloc(IFT_ETHER); 633270063Sluigi if (ifp == NULL) { 634343834Svmaffione nm_prerr("if_alloc failed"); 635270063Sluigi return ENOMEM; 636270063Sluigi } 637270063Sluigi if_initname(ifp, name, IF_DUNIT_NONE); 638270063Sluigi ifp->if_mtu = 65536; 639270063Sluigi ifp->if_flags = IFF_UP | IFF_SIMPLEX | IFF_MULTICAST; 640270063Sluigi ifp->if_init = (void *)nm_vi_dummy; 641270063Sluigi ifp->if_ioctl = nm_vi_dummy; 642270063Sluigi ifp->if_start = nm_vi_start; 643270063Sluigi ifp->if_mtu = ETHERMTU; 644270063Sluigi IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen); 645270063Sluigi ifp->if_capabilities |= IFCAP_LINKSTATE; 646270063Sluigi ifp->if_capenable |= IFCAP_LINKSTATE; 647270063Sluigi 648270063Sluigi ether_ifattach(ifp, eaddr); 649270063Sluigi *ret = ifp; 650270063Sluigi return 0; 651270063Sluigi} 652341477Svmaffione 653270063Sluigi/* unregister from the system and drop the final refcount */ 654270063Sluigivoid 655341477Svmaffionenm_os_vi_detach(struct ifnet *ifp) 656270063Sluigi{ 657270063Sluigi nm_vi_free_index(((char *)IF_LLADDR(ifp))[5]); 658270063Sluigi ether_ifdetach(ifp); 659270063Sluigi if_free(ifp); 660270063Sluigi} 661270063Sluigi 662341477Svmaffione#ifdef WITH_EXTMEM 663341477Svmaffione#include <vm/vm_map.h> 664369514Sgit2svn#include <vm/vm_extern.h> 665341477Svmaffione#include <vm/vm_kern.h> 666341477Svmaffionestruct nm_os_extmem { 667341477Svmaffione vm_object_t obj; 668341477Svmaffione vm_offset_t kva; 669341477Svmaffione vm_offset_t size; 670341477Svmaffione uintptr_t scan; 671341477Svmaffione}; 672341477Svmaffione 673341477Svmaffionevoid 674341477Svmaffionenm_os_extmem_delete(struct nm_os_extmem *e) 675341477Svmaffione{ 676343834Svmaffione nm_prinf("freeing %zx bytes", (size_t)e->size); 677341477Svmaffione vm_map_remove(kernel_map, e->kva, e->kva + e->size); 678341477Svmaffione nm_os_free(e); 679341477Svmaffione} 680341477Svmaffione 681341477Svmaffionechar * 682341477Svmaffionenm_os_extmem_nextpage(struct nm_os_extmem *e) 683341477Svmaffione{ 684341477Svmaffione char *rv = NULL; 685341477Svmaffione if (e->scan < e->kva + e->size) { 686341477Svmaffione rv = (char *)e->scan; 687341477Svmaffione e->scan += PAGE_SIZE; 688341477Svmaffione } 689341477Svmaffione return rv; 690341477Svmaffione} 691341477Svmaffione 692341477Svmaffioneint 693341477Svmaffionenm_os_extmem_isequal(struct nm_os_extmem *e1, struct nm_os_extmem *e2) 694341477Svmaffione{ 695341477Svmaffione return (e1->obj == e2->obj); 696341477Svmaffione} 697341477Svmaffione 698341477Svmaffioneint 699341477Svmaffionenm_os_extmem_nr_pages(struct nm_os_extmem *e) 700341477Svmaffione{ 701341477Svmaffione return e->size >> PAGE_SHIFT; 702341477Svmaffione} 703341477Svmaffione 704341477Svmaffionestruct nm_os_extmem * 705341477Svmaffionenm_os_extmem_create(unsigned long p, struct nmreq_pools_info *pi, int *perror) 706341477Svmaffione{ 707341477Svmaffione vm_map_t map; 708341477Svmaffione vm_map_entry_t entry; 709341477Svmaffione vm_object_t obj; 710341477Svmaffione vm_prot_t prot; 711341477Svmaffione vm_pindex_t index; 712341477Svmaffione boolean_t wired; 713341477Svmaffione struct nm_os_extmem *e = NULL; 714341477Svmaffione int rv, error = 0; 715341477Svmaffione 716341477Svmaffione e = nm_os_malloc(sizeof(*e)); 717341477Svmaffione if (e == NULL) { 718341477Svmaffione error = ENOMEM; 719341477Svmaffione goto out; 720341477Svmaffione } 721341477Svmaffione 722341477Svmaffione map = &curthread->td_proc->p_vmspace->vm_map; 723341477Svmaffione rv = vm_map_lookup(&map, p, VM_PROT_RW, &entry, 724341477Svmaffione &obj, &index, &prot, &wired); 725341477Svmaffione if (rv != KERN_SUCCESS) { 726343834Svmaffione nm_prerr("address %lx not found", p); 727369514Sgit2svn error = vm_mmap_to_errno(rv); 728341477Svmaffione goto out_free; 729341477Svmaffione } 730369514Sgit2svn vm_object_reference(obj); 731369514Sgit2svn 732341477Svmaffione /* check that we are given the whole vm_object ? */ 733341477Svmaffione vm_map_lookup_done(map, entry); 734341477Svmaffione 735341477Svmaffione e->obj = obj; 736369514Sgit2svn /* Wire the memory and add the vm_object to the kernel map, 737369514Sgit2svn * to make sure that it is not freed even if all the processes 738369514Sgit2svn * that are mmap()ing should munmap() it. 739341477Svmaffione */ 740341477Svmaffione e->kva = vm_map_min(kernel_map); 741341477Svmaffione e->size = obj->size << PAGE_SHIFT; 742341477Svmaffione rv = vm_map_find(kernel_map, obj, 0, &e->kva, e->size, 0, 743341477Svmaffione VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 744341477Svmaffione VM_PROT_READ | VM_PROT_WRITE, 0); 745341477Svmaffione if (rv != KERN_SUCCESS) { 746343834Svmaffione nm_prerr("vm_map_find(%zx) failed", (size_t)e->size); 747369514Sgit2svn error = vm_mmap_to_errno(rv); 748341477Svmaffione goto out_rel; 749341477Svmaffione } 750341477Svmaffione rv = vm_map_wire(kernel_map, e->kva, e->kva + e->size, 751341477Svmaffione VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); 752341477Svmaffione if (rv != KERN_SUCCESS) { 753343834Svmaffione nm_prerr("vm_map_wire failed"); 754369514Sgit2svn error = vm_mmap_to_errno(rv); 755341477Svmaffione goto out_rem; 756341477Svmaffione } 757341477Svmaffione 758341477Svmaffione e->scan = e->kva; 759341477Svmaffione 760341477Svmaffione return e; 761341477Svmaffione 762341477Svmaffioneout_rem: 763341477Svmaffione vm_map_remove(kernel_map, e->kva, e->kva + e->size); 764341477Svmaffioneout_rel: 765341477Svmaffione vm_object_deallocate(e->obj); 766369514Sgit2svn e->obj = NULL; 767341477Svmaffioneout_free: 768341477Svmaffione nm_os_free(e); 769341477Svmaffioneout: 770341477Svmaffione if (perror) 771341477Svmaffione *perror = error; 772341477Svmaffione return NULL; 773341477Svmaffione} 774341477Svmaffione#endif /* WITH_EXTMEM */ 775341477Svmaffione 776342033Svmaffione/* ================== PTNETMAP GUEST SUPPORT ==================== */ 777341477Svmaffione 778342033Svmaffione#ifdef WITH_PTNETMAP 779341477Svmaffione#include <sys/bus.h> 780341477Svmaffione#include <sys/rman.h> 781341477Svmaffione#include <machine/bus.h> /* bus_dmamap_* */ 782341477Svmaffione#include <machine/resource.h> 783341477Svmaffione#include <dev/pci/pcivar.h> 784341477Svmaffione#include <dev/pci/pcireg.h> 785270063Sluigi/* 786341477Svmaffione * ptnetmap memory device (memdev) for freebsd guest, 787341477Svmaffione * ssed to expose host netmap memory to the guest through a PCI BAR. 788341477Svmaffione */ 789341477Svmaffione 790341477Svmaffione/* 791341477Svmaffione * ptnetmap memdev private data structure 792341477Svmaffione */ 793341477Svmaffionestruct ptnetmap_memdev { 794341477Svmaffione device_t dev; 795341477Svmaffione struct resource *pci_io; 796341477Svmaffione struct resource *pci_mem; 797341477Svmaffione struct netmap_mem_d *nm_mem; 798341477Svmaffione}; 799341477Svmaffione 800341477Svmaffionestatic int ptn_memdev_probe(device_t); 801341477Svmaffionestatic int ptn_memdev_attach(device_t); 802341477Svmaffionestatic int ptn_memdev_detach(device_t); 803341477Svmaffionestatic int ptn_memdev_shutdown(device_t); 804341477Svmaffione 805341477Svmaffionestatic device_method_t ptn_memdev_methods[] = { 806341477Svmaffione DEVMETHOD(device_probe, ptn_memdev_probe), 807341477Svmaffione DEVMETHOD(device_attach, ptn_memdev_attach), 808341477Svmaffione DEVMETHOD(device_detach, ptn_memdev_detach), 809341477Svmaffione DEVMETHOD(device_shutdown, ptn_memdev_shutdown), 810341477Svmaffione DEVMETHOD_END 811341477Svmaffione}; 812341477Svmaffione 813341477Svmaffionestatic driver_t ptn_memdev_driver = { 814341477Svmaffione PTNETMAP_MEMDEV_NAME, 815341477Svmaffione ptn_memdev_methods, 816341477Svmaffione sizeof(struct ptnetmap_memdev), 817341477Svmaffione}; 818341477Svmaffione 819341477Svmaffione/* We use (SI_ORDER_MIDDLE+1) here, see DEV_MODULE_ORDERED() invocation 820341477Svmaffione * below. */ 821341477Svmaffionestatic devclass_t ptnetmap_devclass; 822341477SvmaffioneDRIVER_MODULE_ORDERED(ptn_memdev, pci, ptn_memdev_driver, ptnetmap_devclass, 823341477Svmaffione NULL, NULL, SI_ORDER_MIDDLE + 1); 824341477Svmaffione 825341477Svmaffione/* 826341477Svmaffione * Map host netmap memory through PCI-BAR in the guest OS, 827341477Svmaffione * returning physical (nm_paddr) and virtual (nm_addr) addresses 828341477Svmaffione * of the netmap memory mapped in the guest. 829341477Svmaffione */ 830341477Svmaffioneint 831341477Svmaffionenm_os_pt_memdev_iomap(struct ptnetmap_memdev *ptn_dev, vm_paddr_t *nm_paddr, 832341477Svmaffione void **nm_addr, uint64_t *mem_size) 833341477Svmaffione{ 834341477Svmaffione int rid; 835341477Svmaffione 836343834Svmaffione nm_prinf("ptn_memdev_driver iomap"); 837341477Svmaffione 838341477Svmaffione rid = PCIR_BAR(PTNETMAP_MEM_PCI_BAR); 839341477Svmaffione *mem_size = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMSIZE_HI); 840341477Svmaffione *mem_size = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMSIZE_LO) | 841341477Svmaffione (*mem_size << 32); 842341477Svmaffione 843341477Svmaffione /* map memory allocator */ 844341477Svmaffione ptn_dev->pci_mem = bus_alloc_resource(ptn_dev->dev, SYS_RES_MEMORY, 845341477Svmaffione &rid, 0, ~0, *mem_size, RF_ACTIVE); 846341477Svmaffione if (ptn_dev->pci_mem == NULL) { 847341477Svmaffione *nm_paddr = 0; 848341477Svmaffione *nm_addr = NULL; 849341477Svmaffione return ENOMEM; 850341477Svmaffione } 851341477Svmaffione 852341477Svmaffione *nm_paddr = rman_get_start(ptn_dev->pci_mem); 853341477Svmaffione *nm_addr = rman_get_virtual(ptn_dev->pci_mem); 854341477Svmaffione 855343834Svmaffione nm_prinf("=== BAR %d start %lx len %lx mem_size %lx ===", 856341477Svmaffione PTNETMAP_MEM_PCI_BAR, 857341477Svmaffione (unsigned long)(*nm_paddr), 858341477Svmaffione (unsigned long)rman_get_size(ptn_dev->pci_mem), 859341477Svmaffione (unsigned long)*mem_size); 860341477Svmaffione return (0); 861341477Svmaffione} 862341477Svmaffione 863341477Svmaffioneuint32_t 864341477Svmaffionenm_os_pt_memdev_ioread(struct ptnetmap_memdev *ptn_dev, unsigned int reg) 865341477Svmaffione{ 866341477Svmaffione return bus_read_4(ptn_dev->pci_io, reg); 867341477Svmaffione} 868341477Svmaffione 869341477Svmaffione/* Unmap host netmap memory. */ 870341477Svmaffionevoid 871341477Svmaffionenm_os_pt_memdev_iounmap(struct ptnetmap_memdev *ptn_dev) 872341477Svmaffione{ 873343834Svmaffione nm_prinf("ptn_memdev_driver iounmap"); 874341477Svmaffione 875341477Svmaffione if (ptn_dev->pci_mem) { 876341477Svmaffione bus_release_resource(ptn_dev->dev, SYS_RES_MEMORY, 877341477Svmaffione PCIR_BAR(PTNETMAP_MEM_PCI_BAR), ptn_dev->pci_mem); 878341477Svmaffione ptn_dev->pci_mem = NULL; 879341477Svmaffione } 880341477Svmaffione} 881341477Svmaffione 882341477Svmaffione/* Device identification routine, return BUS_PROBE_DEFAULT on success, 883341477Svmaffione * positive on failure */ 884341477Svmaffionestatic int 885341477Svmaffioneptn_memdev_probe(device_t dev) 886341477Svmaffione{ 887341477Svmaffione char desc[256]; 888341477Svmaffione 889341477Svmaffione if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID) 890341477Svmaffione return (ENXIO); 891341477Svmaffione if (pci_get_device(dev) != PTNETMAP_PCI_DEVICE_ID) 892341477Svmaffione return (ENXIO); 893341477Svmaffione 894341477Svmaffione snprintf(desc, sizeof(desc), "%s PCI adapter", 895341477Svmaffione PTNETMAP_MEMDEV_NAME); 896341477Svmaffione device_set_desc_copy(dev, desc); 897341477Svmaffione 898341477Svmaffione return (BUS_PROBE_DEFAULT); 899341477Svmaffione} 900341477Svmaffione 901341477Svmaffione/* Device initialization routine. */ 902341477Svmaffionestatic int 903341477Svmaffioneptn_memdev_attach(device_t dev) 904341477Svmaffione{ 905341477Svmaffione struct ptnetmap_memdev *ptn_dev; 906341477Svmaffione int rid; 907341477Svmaffione uint16_t mem_id; 908341477Svmaffione 909341477Svmaffione ptn_dev = device_get_softc(dev); 910341477Svmaffione ptn_dev->dev = dev; 911341477Svmaffione 912341477Svmaffione pci_enable_busmaster(dev); 913341477Svmaffione 914341477Svmaffione rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR); 915341477Svmaffione ptn_dev->pci_io = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 916341477Svmaffione RF_ACTIVE); 917341477Svmaffione if (ptn_dev->pci_io == NULL) { 918341477Svmaffione device_printf(dev, "cannot map I/O space\n"); 919341477Svmaffione return (ENXIO); 920341477Svmaffione } 921341477Svmaffione 922341477Svmaffione mem_id = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMID); 923341477Svmaffione 924341477Svmaffione /* create guest allocator */ 925341477Svmaffione ptn_dev->nm_mem = netmap_mem_pt_guest_attach(ptn_dev, mem_id); 926341477Svmaffione if (ptn_dev->nm_mem == NULL) { 927341477Svmaffione ptn_memdev_detach(dev); 928341477Svmaffione return (ENOMEM); 929341477Svmaffione } 930341477Svmaffione netmap_mem_get(ptn_dev->nm_mem); 931341477Svmaffione 932343834Svmaffione nm_prinf("ptnetmap memdev attached, host memid: %u", mem_id); 933341477Svmaffione 934341477Svmaffione return (0); 935341477Svmaffione} 936341477Svmaffione 937341477Svmaffione/* Device removal routine. */ 938341477Svmaffionestatic int 939341477Svmaffioneptn_memdev_detach(device_t dev) 940341477Svmaffione{ 941341477Svmaffione struct ptnetmap_memdev *ptn_dev; 942341477Svmaffione 943341477Svmaffione ptn_dev = device_get_softc(dev); 944341477Svmaffione 945341477Svmaffione if (ptn_dev->nm_mem) { 946343834Svmaffione nm_prinf("ptnetmap memdev detached, host memid %u", 947343834Svmaffione netmap_mem_get_id(ptn_dev->nm_mem)); 948341477Svmaffione netmap_mem_put(ptn_dev->nm_mem); 949341477Svmaffione ptn_dev->nm_mem = NULL; 950341477Svmaffione } 951341477Svmaffione if (ptn_dev->pci_mem) { 952341477Svmaffione bus_release_resource(dev, SYS_RES_MEMORY, 953341477Svmaffione PCIR_BAR(PTNETMAP_MEM_PCI_BAR), ptn_dev->pci_mem); 954341477Svmaffione ptn_dev->pci_mem = NULL; 955341477Svmaffione } 956341477Svmaffione if (ptn_dev->pci_io) { 957341477Svmaffione bus_release_resource(dev, SYS_RES_IOPORT, 958341477Svmaffione PCIR_BAR(PTNETMAP_IO_PCI_BAR), ptn_dev->pci_io); 959341477Svmaffione ptn_dev->pci_io = NULL; 960341477Svmaffione } 961341477Svmaffione 962341477Svmaffione return (0); 963341477Svmaffione} 964341477Svmaffione 965341477Svmaffionestatic int 966341477Svmaffioneptn_memdev_shutdown(device_t dev) 967341477Svmaffione{ 968341477Svmaffione return bus_generic_shutdown(dev); 969341477Svmaffione} 970341477Svmaffione 971342033Svmaffione#endif /* WITH_PTNETMAP */ 972341477Svmaffione 973341477Svmaffione/* 974259412Sluigi * In order to track whether pages are still mapped, we hook into 975259412Sluigi * the standard cdev_pager and intercept the constructor and 976259412Sluigi * destructor. 977259412Sluigi */ 978259412Sluigi 979259412Sluigistruct netmap_vm_handle_t { 980259412Sluigi struct cdev *dev; 981259412Sluigi struct netmap_priv_d *priv; 982259412Sluigi}; 983259412Sluigi 984260368Sluigi 985259412Sluigistatic int 986259412Sluiginetmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 987341477Svmaffione vm_ooffset_t foff, struct ucred *cred, u_short *color) 988259412Sluigi{ 989259412Sluigi struct netmap_vm_handle_t *vmh = handle; 990261909Sluigi 991261909Sluigi if (netmap_verbose) 992343834Svmaffione nm_prinf("handle %p size %jd prot %d foff %jd", 993261909Sluigi handle, (intmax_t)size, prot, (intmax_t)foff); 994274354Sluigi if (color) 995274354Sluigi *color = 0; 996259412Sluigi dev_ref(vmh->dev); 997259412Sluigi return 0; 998259412Sluigi} 999259412Sluigi 1000259412Sluigi 1001259412Sluigistatic void 1002259412Sluiginetmap_dev_pager_dtor(void *handle) 1003259412Sluigi{ 1004259412Sluigi struct netmap_vm_handle_t *vmh = handle; 1005259412Sluigi struct cdev *dev = vmh->dev; 1006259412Sluigi struct netmap_priv_d *priv = vmh->priv; 1007261909Sluigi 1008261909Sluigi if (netmap_verbose) 1009343834Svmaffione nm_prinf("handle %p", handle); 1010259412Sluigi netmap_dtor(priv); 1011259412Sluigi free(vmh, M_DEVBUF); 1012259412Sluigi dev_rel(dev); 1013259412Sluigi} 1014259412Sluigi 1015260368Sluigi 1016259412Sluigistatic int 1017259412Sluiginetmap_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, 1018259412Sluigi int prot, vm_page_t *mres) 1019259412Sluigi{ 1020259412Sluigi struct netmap_vm_handle_t *vmh = object->handle; 1021259412Sluigi struct netmap_priv_d *priv = vmh->priv; 1022285349Sluigi struct netmap_adapter *na = priv->np_na; 1023259412Sluigi vm_paddr_t paddr; 1024259412Sluigi vm_page_t page; 1025259412Sluigi vm_memattr_t memattr; 1026259412Sluigi vm_pindex_t pidx; 1027259412Sluigi 1028343834Svmaffione nm_prdis("object %p offset %jd prot %d mres %p", 1029259412Sluigi object, (intmax_t)offset, prot, mres); 1030259412Sluigi memattr = object->memattr; 1031259412Sluigi pidx = OFF_TO_IDX(offset); 1032285349Sluigi paddr = netmap_mem_ofstophys(na->nm_mem, offset); 1033259412Sluigi if (paddr == 0) 1034259412Sluigi return VM_PAGER_FAIL; 1035259412Sluigi 1036259412Sluigi if (((*mres)->flags & PG_FICTITIOUS) != 0) { 1037259412Sluigi /* 1038259412Sluigi * If the passed in result page is a fake page, update it with 1039259412Sluigi * the new physical address. 1040259412Sluigi */ 1041259412Sluigi page = *mres; 1042259412Sluigi vm_page_updatefake(page, paddr, memattr); 1043259412Sluigi } else { 1044259412Sluigi /* 1045259412Sluigi * Replace the passed in reqpage page with our own fake page and 1046259412Sluigi * free up the all of the original pages. 1047259412Sluigi */ 1048259412Sluigi#ifndef VM_OBJECT_WUNLOCK /* FreeBSD < 10.x */ 1049259412Sluigi#define VM_OBJECT_WUNLOCK VM_OBJECT_UNLOCK 1050259412Sluigi#define VM_OBJECT_WLOCK VM_OBJECT_LOCK 1051259412Sluigi#endif /* VM_OBJECT_WUNLOCK */ 1052259412Sluigi 1053259412Sluigi VM_OBJECT_WUNLOCK(object); 1054259412Sluigi page = vm_page_getfake(paddr, memattr); 1055259412Sluigi VM_OBJECT_WLOCK(object); 1056259412Sluigi vm_page_lock(*mres); 1057259412Sluigi vm_page_free(*mres); 1058259412Sluigi vm_page_unlock(*mres); 1059259412Sluigi *mres = page; 1060259412Sluigi vm_page_insert(page, object, pidx); 1061259412Sluigi } 1062259412Sluigi page->valid = VM_PAGE_BITS_ALL; 1063259412Sluigi return (VM_PAGER_OK); 1064259412Sluigi} 1065259412Sluigi 1066259412Sluigi 1067259412Sluigistatic struct cdev_pager_ops netmap_cdev_pager_ops = { 1068259412Sluigi .cdev_pg_ctor = netmap_dev_pager_ctor, 1069259412Sluigi .cdev_pg_dtor = netmap_dev_pager_dtor, 1070259412Sluigi .cdev_pg_fault = netmap_dev_pager_fault, 1071259412Sluigi}; 1072259412Sluigi 1073259412Sluigi 1074259412Sluigistatic int 1075259412Sluiginetmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff, 1076259412Sluigi vm_size_t objsize, vm_object_t *objp, int prot) 1077259412Sluigi{ 1078259412Sluigi int error; 1079259412Sluigi struct netmap_vm_handle_t *vmh; 1080259412Sluigi struct netmap_priv_d *priv; 1081259412Sluigi vm_object_t obj; 1082259412Sluigi 1083261909Sluigi if (netmap_verbose) 1084343834Svmaffione nm_prinf("cdev %p foff %jd size %jd objp %p prot %d", cdev, 1085261909Sluigi (intmax_t )*foff, (intmax_t )objsize, objp, prot); 1086259412Sluigi 1087259412Sluigi vmh = malloc(sizeof(struct netmap_vm_handle_t), M_DEVBUF, 1088259412Sluigi M_NOWAIT | M_ZERO); 1089259412Sluigi if (vmh == NULL) 1090259412Sluigi return ENOMEM; 1091259412Sluigi vmh->dev = cdev; 1092259412Sluigi 1093259412Sluigi NMG_LOCK(); 1094259412Sluigi error = devfs_get_cdevpriv((void**)&priv); 1095259412Sluigi if (error) 1096259412Sluigi goto err_unlock; 1097285349Sluigi if (priv->np_nifp == NULL) { 1098285349Sluigi error = EINVAL; 1099285349Sluigi goto err_unlock; 1100285349Sluigi } 1101259412Sluigi vmh->priv = priv; 1102285359Sluigi priv->np_refs++; 1103259412Sluigi NMG_UNLOCK(); 1104259412Sluigi 1105259412Sluigi obj = cdev_pager_allocate(vmh, OBJT_DEVICE, 1106259412Sluigi &netmap_cdev_pager_ops, objsize, prot, 1107259412Sluigi *foff, NULL); 1108259412Sluigi if (obj == NULL) { 1109343834Svmaffione nm_prerr("cdev_pager_allocate failed"); 1110259412Sluigi error = EINVAL; 1111259412Sluigi goto err_deref; 1112259412Sluigi } 1113259412Sluigi 1114259412Sluigi *objp = obj; 1115259412Sluigi return 0; 1116259412Sluigi 1117259412Sluigierr_deref: 1118259412Sluigi NMG_LOCK(); 1119285359Sluigi priv->np_refs--; 1120259412Sluigierr_unlock: 1121259412Sluigi NMG_UNLOCK(); 1122259412Sluigi// err: 1123259412Sluigi free(vmh, M_DEVBUF); 1124259412Sluigi return error; 1125259412Sluigi} 1126259412Sluigi 1127285349Sluigi/* 1128285359Sluigi * On FreeBSD the close routine is only called on the last close on 1129285359Sluigi * the device (/dev/netmap) so we cannot do anything useful. 1130285359Sluigi * To track close() on individual file descriptors we pass netmap_dtor() to 1131285349Sluigi * devfs_set_cdevpriv() on open(). The FreeBSD kernel will call the destructor 1132341477Svmaffione * when the last fd pointing to the device is closed. 1133285349Sluigi * 1134285359Sluigi * Note that FreeBSD does not even munmap() on close() so we also have 1135285359Sluigi * to track mmap() ourselves, and postpone the call to 1136285349Sluigi * netmap_dtor() is called when the process has no open fds and no active 1137285349Sluigi * memory maps on /dev/netmap, as in linux. 1138285349Sluigi */ 1139259412Sluigistatic int 1140259412Sluiginetmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 1141259412Sluigi{ 1142259412Sluigi if (netmap_verbose) 1143343834Svmaffione nm_prinf("dev %p fflag 0x%x devtype %d td %p", 1144259412Sluigi dev, fflag, devtype, td); 1145259412Sluigi return 0; 1146259412Sluigi} 1147259412Sluigi 1148259412Sluigi 1149259412Sluigistatic int 1150259412Sluiginetmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 1151259412Sluigi{ 1152259412Sluigi struct netmap_priv_d *priv; 1153259412Sluigi int error; 1154259412Sluigi 1155259412Sluigi (void)dev; 1156259412Sluigi (void)oflags; 1157259412Sluigi (void)devtype; 1158259412Sluigi (void)td; 1159259412Sluigi 1160341477Svmaffione NMG_LOCK(); 1161341477Svmaffione priv = netmap_priv_new(); 1162341477Svmaffione if (priv == NULL) { 1163341477Svmaffione error = ENOMEM; 1164341477Svmaffione goto out; 1165341477Svmaffione } 1166259412Sluigi error = devfs_set_cdevpriv(priv, netmap_dtor); 1167285359Sluigi if (error) { 1168341477Svmaffione netmap_priv_delete(priv); 1169285359Sluigi } 1170341477Svmaffioneout: 1171341477Svmaffione NMG_UNLOCK(); 1172285359Sluigi return error; 1173259412Sluigi} 1174259412Sluigi 1175341477Svmaffione/******************** kthread wrapper ****************/ 1176341477Svmaffione#include <sys/sysproto.h> 1177341477Svmaffioneu_int 1178341477Svmaffionenm_os_ncpus(void) 1179341477Svmaffione{ 1180341477Svmaffione return mp_maxid + 1; 1181341477Svmaffione} 1182341477Svmaffione 1183341477Svmaffionestruct nm_kctx_ctx { 1184342033Svmaffione /* Userspace thread (kthread creator). */ 1185342033Svmaffione struct thread *user_td; 1186341477Svmaffione 1187341477Svmaffione /* worker function and parameter */ 1188341477Svmaffione nm_kctx_worker_fn_t worker_fn; 1189341477Svmaffione void *worker_private; 1190341477Svmaffione 1191341477Svmaffione struct nm_kctx *nmk; 1192341477Svmaffione 1193341477Svmaffione /* integer to manage multiple worker contexts (e.g., RX or TX on ptnetmap) */ 1194341477Svmaffione long type; 1195341477Svmaffione}; 1196341477Svmaffione 1197341477Svmaffionestruct nm_kctx { 1198341477Svmaffione struct thread *worker; 1199341477Svmaffione struct mtx worker_lock; 1200341477Svmaffione struct nm_kctx_ctx worker_ctx; 1201341477Svmaffione int run; /* used to stop kthread */ 1202341477Svmaffione int attach_user; /* kthread attached to user_process */ 1203341477Svmaffione int affinity; 1204341477Svmaffione}; 1205341477Svmaffione 1206341477Svmaffionestatic void 1207341477Svmaffionenm_kctx_worker(void *data) 1208341477Svmaffione{ 1209341477Svmaffione struct nm_kctx *nmk = data; 1210341477Svmaffione struct nm_kctx_ctx *ctx = &nmk->worker_ctx; 1211341477Svmaffione 1212341477Svmaffione if (nmk->affinity >= 0) { 1213341477Svmaffione thread_lock(curthread); 1214341477Svmaffione sched_bind(curthread, nmk->affinity); 1215341477Svmaffione thread_unlock(curthread); 1216341477Svmaffione } 1217341477Svmaffione 1218341477Svmaffione while (nmk->run) { 1219341477Svmaffione /* 1220341477Svmaffione * check if the parent process dies 1221341477Svmaffione * (when kthread is attached to user process) 1222341477Svmaffione */ 1223341477Svmaffione if (ctx->user_td) { 1224341477Svmaffione PROC_LOCK(curproc); 1225341477Svmaffione thread_suspend_check(0); 1226341477Svmaffione PROC_UNLOCK(curproc); 1227341477Svmaffione } else { 1228341477Svmaffione kthread_suspend_check(); 1229341477Svmaffione } 1230341477Svmaffione 1231342033Svmaffione /* Continuously execute worker process. */ 1232342033Svmaffione ctx->worker_fn(ctx->worker_private); /* worker body */ 1233341477Svmaffione } 1234341477Svmaffione 1235341477Svmaffione kthread_exit(); 1236341477Svmaffione} 1237341477Svmaffione 1238341477Svmaffionevoid 1239341477Svmaffionenm_os_kctx_worker_setaff(struct nm_kctx *nmk, int affinity) 1240341477Svmaffione{ 1241341477Svmaffione nmk->affinity = affinity; 1242341477Svmaffione} 1243341477Svmaffione 1244341477Svmaffionestruct nm_kctx * 1245341477Svmaffionenm_os_kctx_create(struct nm_kctx_cfg *cfg, void *opaque) 1246341477Svmaffione{ 1247341477Svmaffione struct nm_kctx *nmk = NULL; 1248341477Svmaffione 1249341477Svmaffione nmk = malloc(sizeof(*nmk), M_DEVBUF, M_NOWAIT | M_ZERO); 1250341477Svmaffione if (!nmk) 1251341477Svmaffione return NULL; 1252341477Svmaffione 1253341477Svmaffione mtx_init(&nmk->worker_lock, "nm_kthread lock", NULL, MTX_DEF); 1254341477Svmaffione nmk->worker_ctx.worker_fn = cfg->worker_fn; 1255341477Svmaffione nmk->worker_ctx.worker_private = cfg->worker_private; 1256341477Svmaffione nmk->worker_ctx.type = cfg->type; 1257341477Svmaffione nmk->affinity = -1; 1258341477Svmaffione 1259341477Svmaffione /* attach kthread to user process (ptnetmap) */ 1260341477Svmaffione nmk->attach_user = cfg->attach_user; 1261341477Svmaffione 1262341477Svmaffione return nmk; 1263341477Svmaffione} 1264341477Svmaffione 1265341477Svmaffioneint 1266341477Svmaffionenm_os_kctx_worker_start(struct nm_kctx *nmk) 1267341477Svmaffione{ 1268341477Svmaffione struct proc *p = NULL; 1269341477Svmaffione int error = 0; 1270341477Svmaffione 1271342033Svmaffione /* Temporarily disable this function as it is currently broken 1272342033Svmaffione * and causes kernel crashes. The failure can be triggered by 1273342033Svmaffione * the "vale_polling_enable_disable" test in ctrl-api-test.c. */ 1274342033Svmaffione return EOPNOTSUPP; 1275342033Svmaffione 1276342033Svmaffione if (nmk->worker) 1277341477Svmaffione return EBUSY; 1278341477Svmaffione 1279341477Svmaffione /* check if we want to attach kthread to user process */ 1280341477Svmaffione if (nmk->attach_user) { 1281341477Svmaffione nmk->worker_ctx.user_td = curthread; 1282341477Svmaffione p = curthread->td_proc; 1283341477Svmaffione } 1284341477Svmaffione 1285341477Svmaffione /* enable kthread main loop */ 1286341477Svmaffione nmk->run = 1; 1287341477Svmaffione /* create kthread */ 1288341477Svmaffione if((error = kthread_add(nm_kctx_worker, nmk, p, 1289341477Svmaffione &nmk->worker, RFNOWAIT /* to be checked */, 0, "nm-kthread-%ld", 1290341477Svmaffione nmk->worker_ctx.type))) { 1291341477Svmaffione goto err; 1292341477Svmaffione } 1293341477Svmaffione 1294343834Svmaffione nm_prinf("nm_kthread started td %p", nmk->worker); 1295341477Svmaffione 1296341477Svmaffione return 0; 1297341477Svmaffioneerr: 1298343834Svmaffione nm_prerr("nm_kthread start failed err %d", error); 1299341477Svmaffione nmk->worker = NULL; 1300341477Svmaffione return error; 1301341477Svmaffione} 1302341477Svmaffione 1303341477Svmaffionevoid 1304341477Svmaffionenm_os_kctx_worker_stop(struct nm_kctx *nmk) 1305341477Svmaffione{ 1306342033Svmaffione if (!nmk->worker) 1307341477Svmaffione return; 1308342033Svmaffione 1309341477Svmaffione /* tell to kthread to exit from main loop */ 1310341477Svmaffione nmk->run = 0; 1311341477Svmaffione 1312341477Svmaffione /* wake up kthread if it sleeps */ 1313341477Svmaffione kthread_resume(nmk->worker); 1314341477Svmaffione 1315341477Svmaffione nmk->worker = NULL; 1316341477Svmaffione} 1317341477Svmaffione 1318341477Svmaffionevoid 1319341477Svmaffionenm_os_kctx_destroy(struct nm_kctx *nmk) 1320341477Svmaffione{ 1321341477Svmaffione if (!nmk) 1322341477Svmaffione return; 1323342033Svmaffione 1324342033Svmaffione if (nmk->worker) 1325341477Svmaffione nm_os_kctx_worker_stop(nmk); 1326341477Svmaffione 1327341477Svmaffione free(nmk, M_DEVBUF); 1328341477Svmaffione} 1329341477Svmaffione 1330261909Sluigi/******************** kqueue support ****************/ 1331259412Sluigi 1332261909Sluigi/* 1333343831Svmaffione * In addition to calling selwakeuppri(), nm_os_selwakeup() also 1334344509Svmaffione * needs to call knote() to wake up kqueue listeners. 1335344509Svmaffione * This operation is deferred to a taskqueue in order to avoid possible 1336344509Svmaffione * lock order reversals; these may happen because knote() grabs a 1337344509Svmaffione * private lock associated to the 'si' (see struct selinfo, 1338344509Svmaffione * struct nm_selinfo, and nm_os_selinfo_init), and nm_os_selwakeup() 1339344509Svmaffione * can be called while holding the lock associated to a different 1340344509Svmaffione * 'si'. 1341344509Svmaffione * When calling knote() we use a non-zero 'hint' argument to inform 1342344509Svmaffione * the netmap_knrw() function that it is being called from 1343344509Svmaffione * 'nm_os_selwakeup'; this is necessary because when netmap_knrw() is 1344344509Svmaffione * called by the kevent subsystem (i.e. kevent_scan()) we also need to 1345344509Svmaffione * call netmap_poll(). 1346261909Sluigi * 1347343831Svmaffione * The netmap_kqfilter() function registers one or another f_event 1348343831Svmaffione * depending on read or write mode. A pointer to the struct 1349343831Svmaffione * 'netmap_priv_d' is stored into kn->kn_hook, so that it can later 1350343831Svmaffione * be passed to netmap_poll(). We pass NULL as a third argument to 1351343831Svmaffione * netmap_poll(), so that the latter only runs the txsync/rxsync 1352343831Svmaffione * (if necessary), and skips the nm_os_selrecord() calls. 1353261909Sluigi */ 1354261909Sluigi 1355261909Sluigi 1356261909Sluigivoid 1357341477Svmaffionenm_os_selwakeup(struct nm_selinfo *si) 1358261909Sluigi{ 1359341477Svmaffione selwakeuppri(&si->si, PI_NET); 1360344509Svmaffione if (si->kqueue_users > 0) { 1361344509Svmaffione taskqueue_enqueue(si->ntfytq, &si->ntfytask); 1362344509Svmaffione } 1363261909Sluigi} 1364261909Sluigi 1365341477Svmaffionevoid 1366341477Svmaffionenm_os_selrecord(struct thread *td, struct nm_selinfo *si) 1367341477Svmaffione{ 1368341477Svmaffione selrecord(td, &si->si); 1369341477Svmaffione} 1370341477Svmaffione 1371261909Sluigistatic void 1372261909Sluiginetmap_knrdetach(struct knote *kn) 1373261909Sluigi{ 1374261909Sluigi struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; 1375344509Svmaffione struct nm_selinfo *si = priv->np_si[NR_RX]; 1376261909Sluigi 1377344509Svmaffione knlist_remove(&si->si.si_note, kn, /*islocked=*/0); 1378344509Svmaffione NMG_LOCK(); 1379344509Svmaffione KASSERT(si->kqueue_users > 0, ("kqueue_user underflow on %s", 1380344509Svmaffione si->mtxname)); 1381344509Svmaffione si->kqueue_users--; 1382344509Svmaffione nm_prinf("kqueue users for %s: %d", si->mtxname, si->kqueue_users); 1383344509Svmaffione NMG_UNLOCK(); 1384261909Sluigi} 1385261909Sluigi 1386261909Sluigistatic void 1387261909Sluiginetmap_knwdetach(struct knote *kn) 1388261909Sluigi{ 1389261909Sluigi struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; 1390344509Svmaffione struct nm_selinfo *si = priv->np_si[NR_TX]; 1391261909Sluigi 1392344509Svmaffione knlist_remove(&si->si.si_note, kn, /*islocked=*/0); 1393344509Svmaffione NMG_LOCK(); 1394344509Svmaffione si->kqueue_users--; 1395344509Svmaffione nm_prinf("kqueue users for %s: %d", si->mtxname, si->kqueue_users); 1396344509Svmaffione NMG_UNLOCK(); 1397261909Sluigi} 1398261909Sluigi 1399261909Sluigi/* 1400343831Svmaffione * Callback triggered by netmap notifications (see netmap_notify()), 1401343831Svmaffione * and by the application calling kevent(). In the former case we 1402343831Svmaffione * just return 1 (events ready), since we are not able to do better. 1403343831Svmaffione * In the latter case we use netmap_poll() to see which events are 1404343831Svmaffione * ready. 1405261909Sluigi */ 1406261909Sluigistatic int 1407261909Sluiginetmap_knrw(struct knote *kn, long hint, int events) 1408261909Sluigi{ 1409261909Sluigi struct netmap_priv_d *priv; 1410261909Sluigi int revents; 1411261909Sluigi 1412261909Sluigi if (hint != 0) { 1413343831Svmaffione /* Called from netmap_notify(), typically from a 1414343831Svmaffione * thread different from the one issuing kevent(). 1415343831Svmaffione * Assume we are ready. */ 1416343831Svmaffione return 1; 1417261909Sluigi } 1418343831Svmaffione 1419343831Svmaffione /* Called from kevent(). */ 1420261909Sluigi priv = kn->kn_hook; 1421343831Svmaffione revents = netmap_poll(priv, events, /*thread=*/NULL); 1422343831Svmaffione 1423343831Svmaffione return (events & revents) ? 1 : 0; 1424261909Sluigi} 1425261909Sluigi 1426261909Sluigistatic int 1427261909Sluiginetmap_knread(struct knote *kn, long hint) 1428261909Sluigi{ 1429261909Sluigi return netmap_knrw(kn, hint, POLLIN); 1430261909Sluigi} 1431261909Sluigi 1432261909Sluigistatic int 1433261909Sluiginetmap_knwrite(struct knote *kn, long hint) 1434261909Sluigi{ 1435261909Sluigi return netmap_knrw(kn, hint, POLLOUT); 1436261909Sluigi} 1437261909Sluigi 1438261909Sluigistatic struct filterops netmap_rfiltops = { 1439261909Sluigi .f_isfd = 1, 1440261909Sluigi .f_detach = netmap_knrdetach, 1441261909Sluigi .f_event = netmap_knread, 1442261909Sluigi}; 1443261909Sluigi 1444261909Sluigistatic struct filterops netmap_wfiltops = { 1445261909Sluigi .f_isfd = 1, 1446261909Sluigi .f_detach = netmap_knwdetach, 1447261909Sluigi .f_event = netmap_knwrite, 1448261909Sluigi}; 1449261909Sluigi 1450261909Sluigi 1451261909Sluigi/* 1452261909Sluigi * This is called when a thread invokes kevent() to record 1453261909Sluigi * a change in the configuration of the kqueue(). 1454343831Svmaffione * The 'priv' is the one associated to the open netmap device. 1455261909Sluigi */ 1456261909Sluigistatic int 1457261909Sluiginetmap_kqfilter(struct cdev *dev, struct knote *kn) 1458261909Sluigi{ 1459261909Sluigi struct netmap_priv_d *priv; 1460261909Sluigi int error; 1461261909Sluigi struct netmap_adapter *na; 1462274459Sluigi struct nm_selinfo *si; 1463261909Sluigi int ev = kn->kn_filter; 1464261909Sluigi 1465261909Sluigi if (ev != EVFILT_READ && ev != EVFILT_WRITE) { 1466343834Svmaffione nm_prerr("bad filter request %d", ev); 1467261909Sluigi return 1; 1468261909Sluigi } 1469261909Sluigi error = devfs_get_cdevpriv((void**)&priv); 1470261909Sluigi if (error) { 1471343834Svmaffione nm_prerr("device not yet setup"); 1472261909Sluigi return 1; 1473261909Sluigi } 1474261909Sluigi na = priv->np_na; 1475261909Sluigi if (na == NULL) { 1476343834Svmaffione nm_prerr("no netmap adapter for this file descriptor"); 1477261909Sluigi return 1; 1478261909Sluigi } 1479261909Sluigi /* the si is indicated in the priv */ 1480285349Sluigi si = priv->np_si[(ev == EVFILT_WRITE) ? NR_TX : NR_RX]; 1481261909Sluigi kn->kn_fop = (ev == EVFILT_WRITE) ? 1482261909Sluigi &netmap_wfiltops : &netmap_rfiltops; 1483261909Sluigi kn->kn_hook = priv; 1484344509Svmaffione NMG_LOCK(); 1485344509Svmaffione si->kqueue_users++; 1486344509Svmaffione nm_prinf("kqueue users for %s: %d", si->mtxname, si->kqueue_users); 1487344509Svmaffione NMG_UNLOCK(); 1488343831Svmaffione knlist_add(&si->si.si_note, kn, /*islocked=*/0); 1489343831Svmaffione 1490261909Sluigi return 0; 1491261909Sluigi} 1492261909Sluigi 1493341477Svmaffionestatic int 1494341477Svmaffionefreebsd_netmap_poll(struct cdev *cdevi __unused, int events, struct thread *td) 1495341477Svmaffione{ 1496341477Svmaffione struct netmap_priv_d *priv; 1497341477Svmaffione if (devfs_get_cdevpriv((void **)&priv)) { 1498341477Svmaffione return POLLERR; 1499341477Svmaffione } 1500341477Svmaffione return netmap_poll(priv, events, td); 1501341477Svmaffione} 1502341477Svmaffione 1503341477Svmaffionestatic int 1504341477Svmaffionefreebsd_netmap_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t data, 1505341477Svmaffione int ffla __unused, struct thread *td) 1506341477Svmaffione{ 1507341477Svmaffione int error; 1508341477Svmaffione struct netmap_priv_d *priv; 1509341477Svmaffione 1510341477Svmaffione CURVNET_SET(TD_TO_VNET(td)); 1511341477Svmaffione error = devfs_get_cdevpriv((void **)&priv); 1512341477Svmaffione if (error) { 1513341477Svmaffione /* XXX ENOENT should be impossible, since the priv 1514341477Svmaffione * is now created in the open */ 1515341477Svmaffione if (error == ENOENT) 1516341477Svmaffione error = ENXIO; 1517341477Svmaffione goto out; 1518341477Svmaffione } 1519341477Svmaffione error = netmap_ioctl(priv, cmd, data, td, /*nr_body_is_user=*/1); 1520341477Svmaffioneout: 1521341477Svmaffione CURVNET_RESTORE(); 1522341477Svmaffione 1523341477Svmaffione return error; 1524341477Svmaffione} 1525341477Svmaffione 1526341477Svmaffionevoid 1527341477Svmaffionenm_os_onattach(struct ifnet *ifp) 1528341477Svmaffione{ 1529341480Svmaffione ifp->if_capabilities |= IFCAP_NETMAP; 1530341477Svmaffione} 1531341477Svmaffione 1532341477Svmaffionevoid 1533341477Svmaffionenm_os_onenter(struct ifnet *ifp) 1534341477Svmaffione{ 1535341477Svmaffione struct netmap_adapter *na = NA(ifp); 1536341477Svmaffione 1537341477Svmaffione na->if_transmit = ifp->if_transmit; 1538341477Svmaffione ifp->if_transmit = netmap_transmit; 1539341477Svmaffione ifp->if_capenable |= IFCAP_NETMAP; 1540341477Svmaffione} 1541341477Svmaffione 1542341477Svmaffionevoid 1543341477Svmaffionenm_os_onexit(struct ifnet *ifp) 1544341477Svmaffione{ 1545341477Svmaffione struct netmap_adapter *na = NA(ifp); 1546341477Svmaffione 1547341477Svmaffione ifp->if_transmit = na->if_transmit; 1548341477Svmaffione ifp->if_capenable &= ~IFCAP_NETMAP; 1549341477Svmaffione} 1550341477Svmaffione 1551341477Svmaffioneextern struct cdevsw netmap_cdevsw; /* XXX used in netmap.c, should go elsewhere */ 1552259412Sluigistruct cdevsw netmap_cdevsw = { 1553259412Sluigi .d_version = D_VERSION, 1554259412Sluigi .d_name = "netmap", 1555259412Sluigi .d_open = netmap_open, 1556259412Sluigi .d_mmap_single = netmap_mmap_single, 1557341477Svmaffione .d_ioctl = freebsd_netmap_ioctl, 1558341477Svmaffione .d_poll = freebsd_netmap_poll, 1559261909Sluigi .d_kqfilter = netmap_kqfilter, 1560259412Sluigi .d_close = netmap_close, 1561259412Sluigi}; 1562261909Sluigi/*--- end of kqueue support ----*/ 1563259412Sluigi 1564259412Sluigi/* 1565259412Sluigi * Kernel entry point. 1566259412Sluigi * 1567259412Sluigi * Initialize/finalize the module and return. 1568259412Sluigi * 1569259412Sluigi * Return 0 on success, errno on failure. 1570259412Sluigi */ 1571259412Sluigistatic int 1572259412Sluiginetmap_loader(__unused struct module *module, int event, __unused void *arg) 1573259412Sluigi{ 1574259412Sluigi int error = 0; 1575259412Sluigi 1576259412Sluigi switch (event) { 1577259412Sluigi case MOD_LOAD: 1578259412Sluigi error = netmap_init(); 1579259412Sluigi break; 1580259412Sluigi 1581259412Sluigi case MOD_UNLOAD: 1582285699Sluigi /* 1583285699Sluigi * if some one is still using netmap, 1584285699Sluigi * then the module can not be unloaded. 1585285699Sluigi */ 1586285699Sluigi if (netmap_use_count) { 1587343834Svmaffione nm_prerr("netmap module can not be unloaded - netmap_use_count: %d", 1588285699Sluigi netmap_use_count); 1589285699Sluigi error = EBUSY; 1590285699Sluigi break; 1591285699Sluigi } 1592259412Sluigi netmap_fini(); 1593259412Sluigi break; 1594259412Sluigi 1595259412Sluigi default: 1596259412Sluigi error = EOPNOTSUPP; 1597259412Sluigi break; 1598259412Sluigi } 1599259412Sluigi 1600259412Sluigi return (error); 1601259412Sluigi} 1602259412Sluigi 1603341477Svmaffione#ifdef DEV_MODULE_ORDERED 1604341477Svmaffione/* 1605341477Svmaffione * The netmap module contains three drivers: (i) the netmap character device 1606341477Svmaffione * driver; (ii) the ptnetmap memdev PCI device driver, (iii) the ptnet PCI 1607341477Svmaffione * device driver. The attach() routines of both (ii) and (iii) need the 1608341477Svmaffione * lock of the global allocator, and such lock is initialized in netmap_init(), 1609341477Svmaffione * which is part of (i). 1610341477Svmaffione * Therefore, we make sure that (i) is loaded before (ii) and (iii), using 1611341477Svmaffione * the 'order' parameter of driver declaration macros. For (i), we specify 1612341477Svmaffione * SI_ORDER_MIDDLE, while higher orders are used with the DRIVER_MODULE_ORDERED 1613341477Svmaffione * macros for (ii) and (iii). 1614341477Svmaffione */ 1615341477SvmaffioneDEV_MODULE_ORDERED(netmap, netmap_loader, NULL, SI_ORDER_MIDDLE); 1616341477Svmaffione#else /* !DEV_MODULE_ORDERED */ 1617259412SluigiDEV_MODULE(netmap, netmap_loader, NULL); 1618341477Svmaffione#endif /* DEV_MODULE_ORDERED */ 1619341477SvmaffioneMODULE_DEPEND(netmap, pci, 1, 1, 1); 1620279199SluigiMODULE_VERSION(netmap, 1); 1621341477Svmaffione/* reduce conditional code */ 1622341477Svmaffione// linux API, use for the knlist in FreeBSD 1623341477Svmaffione/* use a private mutex for the knlist */ 1624