1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include "opt_inet.h" 29#include "opt_inet6.h" 30 31#include <sys/param.h> 32#include <sys/module.h> 33#include <sys/errno.h> 34#include <sys/eventhandler.h> 35#include <sys/jail.h> 36#include <sys/poll.h> /* POLLIN, POLLOUT */ 37#include <sys/kernel.h> /* types used in module initialization */ 38#include <sys/conf.h> /* DEV_MODULE_ORDERED */ 39#include <sys/endian.h> 40#include <sys/syscallsubr.h> /* kern_ioctl() */ 41 42#include <sys/rwlock.h> 43 44#include <vm/vm.h> /* vtophys */ 45#include <vm/pmap.h> /* vtophys */ 46#include <vm/vm_param.h> 47#include <vm/vm_object.h> 48#include <vm/vm_page.h> 49#include <vm/vm_pager.h> 50#include <vm/uma.h> 51 52 53#include <sys/malloc.h> 54#include <sys/socket.h> /* sockaddrs */ 55#include <sys/selinfo.h> 56#include <sys/kthread.h> /* kthread_add() */ 57#include <sys/proc.h> /* PROC_LOCK() */ 58#include <sys/unistd.h> /* RFNOWAIT */ 59#include <sys/sched.h> /* sched_bind() */ 60#include <sys/smp.h> /* mp_maxid */ 61#include <sys/taskqueue.h> /* taskqueue_enqueue(), taskqueue_create(), ... */ 62#include <net/if.h> 63#include <net/if_var.h> 64#include <net/if_types.h> /* IFT_ETHER */ 65#include <net/ethernet.h> /* ether_ifdetach */ 66#include <net/if_dl.h> /* LLADDR */ 67#include <machine/bus.h> /* bus_dmamap_* */ 68#include <netinet/in.h> /* in6_cksum_pseudo() */ 69#include <machine/in_cksum.h> /* in_pseudo(), in_cksum_hdr() */ 70 71#include <net/netmap.h> 72#include <dev/netmap/netmap_kern.h> 73#include <net/netmap_virt.h> 74#include <dev/netmap/netmap_mem2.h> 75 76 77/* ======================== FREEBSD-SPECIFIC ROUTINES ================== */ 78 79static void 80nm_kqueue_notify(void *opaque, int pending) 81{ 82 struct nm_selinfo *si = opaque; 83 84 /* We use a non-zero hint to distinguish this notification call 85 * from the call done in kqueue_scan(), which uses hint=0. 86 */ 87 KNOTE_UNLOCKED(&si->si.si_note, /*hint=*/0x100); 88} 89 90int nm_os_selinfo_init(NM_SELINFO_T *si, const char *name) { 91 int err; 92 93 TASK_INIT(&si->ntfytask, 0, nm_kqueue_notify, si); 94 si->ntfytq = taskqueue_create(name, M_NOWAIT, 95 taskqueue_thread_enqueue, &si->ntfytq); 96 if (si->ntfytq == NULL) 97 return -ENOMEM; 98 err = taskqueue_start_threads(&si->ntfytq, 1, PI_NET, "tq %s", name); 99 if (err) { 100 taskqueue_free(si->ntfytq); 101 si->ntfytq = NULL; 102 return err; 103 } 104 105 snprintf(si->mtxname, sizeof(si->mtxname), "nmkl%s", name); 106 mtx_init(&si->m, si->mtxname, NULL, MTX_DEF); 107 knlist_init_mtx(&si->si.si_note, &si->m); 108 si->kqueue_users = 0; 109 110 return (0); 111} 112 113void 114nm_os_selinfo_uninit(NM_SELINFO_T *si) 115{ 116 if (si->ntfytq == NULL) { 117 return; /* si was not initialized */ 118 } 119 taskqueue_drain(si->ntfytq, &si->ntfytask); 120 taskqueue_free(si->ntfytq); 121 si->ntfytq = NULL; 122 knlist_delete(&si->si.si_note, curthread, /*islocked=*/0); 123 knlist_destroy(&si->si.si_note); 124 /* now we don't need the mutex anymore */ 125 mtx_destroy(&si->m); 126} 127 128void * 129nm_os_malloc(size_t size) 130{ 131 return malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 132} 133 134void * 135nm_os_realloc(void *addr, size_t new_size, size_t old_size __unused) 136{ 137 return realloc(addr, new_size, M_DEVBUF, M_NOWAIT | M_ZERO); 138} 139 140void 141nm_os_free(void *addr) 142{ 143 free(addr, M_DEVBUF); 144} 145 146void 147nm_os_ifnet_lock(void) 148{ 149 IFNET_RLOCK(); 150} 151 152void 153nm_os_ifnet_unlock(void) 154{ 155 IFNET_RUNLOCK(); 156} 157 158static int netmap_use_count = 0; 159 160void 161nm_os_get_module(void) 162{ 163 netmap_use_count++; 164} 165 166void 167nm_os_put_module(void) 168{ 169 netmap_use_count--; 170} 171 172static void 173netmap_ifnet_arrival_handler(void *arg __unused, if_t ifp) 174{ 175 netmap_undo_zombie(ifp); 176} 177 178static void 179netmap_ifnet_departure_handler(void *arg __unused, if_t ifp) 180{ 181 netmap_make_zombie(ifp); 182} 183 184static eventhandler_tag nm_ifnet_ah_tag; 185static eventhandler_tag nm_ifnet_dh_tag; 186 187int 188nm_os_ifnet_init(void) 189{ 190 nm_ifnet_ah_tag = 191 EVENTHANDLER_REGISTER(ifnet_arrival_event, 192 netmap_ifnet_arrival_handler, 193 NULL, EVENTHANDLER_PRI_ANY); 194 nm_ifnet_dh_tag = 195 EVENTHANDLER_REGISTER(ifnet_departure_event, 196 netmap_ifnet_departure_handler, 197 NULL, EVENTHANDLER_PRI_ANY); 198 return 0; 199} 200 201void 202nm_os_ifnet_fini(void) 203{ 204 EVENTHANDLER_DEREGISTER(ifnet_arrival_event, 205 nm_ifnet_ah_tag); 206 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 207 nm_ifnet_dh_tag); 208} 209 210unsigned 211nm_os_ifnet_mtu(if_t ifp) 212{ 213 return if_getmtu(ifp); 214} 215 216rawsum_t 217nm_os_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum) 218{ 219 /* TODO XXX please use the FreeBSD implementation for this. */ 220 uint16_t *words = (uint16_t *)data; 221 int nw = len / 2; 222 int i; 223 224 for (i = 0; i < nw; i++) 225 cur_sum += be16toh(words[i]); 226 227 if (len & 1) 228 cur_sum += (data[len-1] << 8); 229 230 return cur_sum; 231} 232 233/* Fold a raw checksum: 'cur_sum' is in host byte order, while the 234 * return value is in network byte order. 235 */ 236uint16_t 237nm_os_csum_fold(rawsum_t cur_sum) 238{ 239 /* TODO XXX please use the FreeBSD implementation for this. */ 240 while (cur_sum >> 16) 241 cur_sum = (cur_sum & 0xFFFF) + (cur_sum >> 16); 242 243 return htobe16((~cur_sum) & 0xFFFF); 244} 245 246uint16_t nm_os_csum_ipv4(struct nm_iphdr *iph) 247{ 248#if 0 249 return in_cksum_hdr((void *)iph); 250#else 251 return nm_os_csum_fold(nm_os_csum_raw((uint8_t*)iph, sizeof(struct nm_iphdr), 0)); 252#endif 253} 254 255void 256nm_os_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, 257 size_t datalen, uint16_t *check) 258{ 259#ifdef INET 260 uint16_t pseudolen = datalen + iph->protocol; 261 262 /* Compute and insert the pseudo-header checksum. */ 263 *check = in_pseudo(iph->saddr, iph->daddr, 264 htobe16(pseudolen)); 265 /* Compute the checksum on TCP/UDP header + payload 266 * (includes the pseudo-header). 267 */ 268 *check = nm_os_csum_fold(nm_os_csum_raw(data, datalen, 0)); 269#else 270 static int notsupported = 0; 271 if (!notsupported) { 272 notsupported = 1; 273 nm_prerr("inet4 segmentation not supported"); 274 } 275#endif 276} 277 278void 279nm_os_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, 280 size_t datalen, uint16_t *check) 281{ 282#ifdef INET6 283 *check = in6_cksum_pseudo((void*)ip6h, datalen, ip6h->nexthdr, 0); 284 *check = nm_os_csum_fold(nm_os_csum_raw(data, datalen, 0)); 285#else 286 static int notsupported = 0; 287 if (!notsupported) { 288 notsupported = 1; 289 nm_prerr("inet6 segmentation not supported"); 290 } 291#endif 292} 293 294/* on FreeBSD we send up one packet at a time */ 295void * 296nm_os_send_up(if_t ifp, struct mbuf *m, struct mbuf *prev) 297{ 298 NA(ifp)->if_input(ifp, m); 299 return NULL; 300} 301 302int 303nm_os_mbuf_has_csum_offld(struct mbuf *m) 304{ 305 return m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_SCTP | 306 CSUM_TCP_IPV6 | CSUM_UDP_IPV6 | 307 CSUM_SCTP_IPV6); 308} 309 310int 311nm_os_mbuf_has_seg_offld(struct mbuf *m) 312{ 313 return m->m_pkthdr.csum_flags & CSUM_TSO; 314} 315 316static void 317freebsd_generic_rx_handler(if_t ifp, struct mbuf *m) 318{ 319 int stolen; 320 321 if (unlikely(!NM_NA_VALID(ifp))) { 322 nm_prlim(1, "Warning: RX packet intercepted, but no" 323 " emulated adapter"); 324 return; 325 } 326 327 do { 328 struct mbuf *n; 329 330 n = m->m_nextpkt; 331 m->m_nextpkt = NULL; 332 stolen = generic_rx_handler(ifp, m); 333 if (!stolen) { 334 NA(ifp)->if_input(ifp, m); 335 } 336 m = n; 337 } while (m != NULL); 338} 339 340/* 341 * Intercept the rx routine in the standard device driver. 342 * Second argument is non-zero to intercept, 0 to restore 343 */ 344int 345nm_os_catch_rx(struct netmap_generic_adapter *gna, int intercept) 346{ 347 struct netmap_adapter *na = &gna->up.up; 348 if_t ifp = na->ifp; 349 int ret = 0; 350 351 nm_os_ifnet_lock(); 352 if (intercept) { 353 if_setcapenablebit(ifp, IFCAP_NETMAP, 0); 354 if_setinputfn(ifp, freebsd_generic_rx_handler); 355 } else { 356 if_setcapenablebit(ifp, 0, IFCAP_NETMAP); 357 if_setinputfn(ifp, na->if_input); 358 } 359 nm_os_ifnet_unlock(); 360 361 return ret; 362} 363 364 365/* 366 * Intercept the packet steering routine in the tx path, 367 * so that we can decide which queue is used for an mbuf. 368 * Second argument is non-zero to intercept, 0 to restore. 369 * On freebsd we just intercept if_transmit. 370 */ 371int 372nm_os_catch_tx(struct netmap_generic_adapter *gna, int intercept) 373{ 374 struct netmap_adapter *na = &gna->up.up; 375 if_t ifp = netmap_generic_getifp(gna); 376 377 nm_os_ifnet_lock(); 378 if (intercept) { 379 na->if_transmit = if_gettransmitfn(ifp); 380 if_settransmitfn(ifp, netmap_transmit); 381 } else { 382 if_settransmitfn(ifp, na->if_transmit); 383 } 384 nm_os_ifnet_unlock(); 385 386 return 0; 387} 388 389 390/* 391 * Transmit routine used by generic_netmap_txsync(). Returns 0 on success 392 * and non-zero on error (which may be packet drops or other errors). 393 * addr and len identify the netmap buffer, m is the (preallocated) 394 * mbuf to use for transmissions. 395 * 396 * Zero-copy transmission is possible if netmap is attached directly to a 397 * hardware interface: when cleaning we simply wait for the mbuf cluster 398 * refcount to decrement to 1, indicating that the driver has completed 399 * transmission and is done with the buffer. However, this approach can 400 * lead to queue deadlocks when attaching to software interfaces (e.g., 401 * if_bridge) since we cannot rely on member ports to promptly reclaim 402 * transmitted mbufs. Since there is no easy way to distinguish these 403 * cases, we currently always copy the buffer. 404 * 405 * On multiqueue cards, we can force the queue using 406 * if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 407 * i = m->m_pkthdr.flowid % adapter->num_queues; 408 * else 409 * i = curcpu % adapter->num_queues; 410 */ 411int 412nm_os_generic_xmit_frame(struct nm_os_gen_arg *a) 413{ 414 int ret; 415 u_int len = a->len; 416 if_t ifp = a->ifp; 417 struct mbuf *m = a->m; 418 419 M_ASSERTPKTHDR(m); 420 KASSERT((m->m_flags & M_EXT) != 0, 421 ("%s: mbuf %p has no cluster", __func__, m)); 422 423 if (MBUF_REFCNT(m) != 1) { 424 nm_prerr("invalid refcnt %d for %p", MBUF_REFCNT(m), m); 425 panic("in generic_xmit_frame"); 426 } 427 if (unlikely(m->m_ext.ext_size < len)) { 428 nm_prlim(2, "size %d < len %d", m->m_ext.ext_size, len); 429 len = m->m_ext.ext_size; 430 } 431 432 m_copyback(m, 0, len, a->addr); 433 m->m_len = m->m_pkthdr.len = len; 434 SET_MBUF_REFCNT(m, 2); 435 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 436 m->m_pkthdr.flowid = a->ring_nr; 437 m->m_pkthdr.rcvif = ifp; /* used for tx notification */ 438 CURVNET_SET(if_getvnet(ifp)); 439 ret = NA(ifp)->if_transmit(ifp, m); 440 CURVNET_RESTORE(); 441 return ret ? -1 : 0; 442} 443 444struct netmap_adapter * 445netmap_getna(if_t ifp) 446{ 447 return (NA(ifp)); 448} 449 450/* 451 * The following two functions are empty until we have a generic 452 * way to extract the info from the ifp 453 */ 454int 455nm_os_generic_find_num_desc(if_t ifp, unsigned int *tx, unsigned int *rx) 456{ 457 return 0; 458} 459 460 461void 462nm_os_generic_find_num_queues(if_t ifp, u_int *txq, u_int *rxq) 463{ 464 unsigned num_rings = netmap_generic_rings ? netmap_generic_rings : 1; 465 466 *txq = num_rings; 467 *rxq = num_rings; 468} 469 470void 471nm_os_generic_set_features(struct netmap_generic_adapter *gna) 472{ 473 474 gna->rxsg = 1; /* Supported through m_copydata. */ 475 gna->txqdisc = 0; /* Not supported. */ 476} 477 478void 479nm_os_mitigation_init(struct nm_generic_mit *mit, int idx, struct netmap_adapter *na) 480{ 481 mit->mit_pending = 0; 482 mit->mit_ring_idx = idx; 483 mit->mit_na = na; 484} 485 486 487void 488nm_os_mitigation_start(struct nm_generic_mit *mit) 489{ 490} 491 492 493void 494nm_os_mitigation_restart(struct nm_generic_mit *mit) 495{ 496} 497 498 499int 500nm_os_mitigation_active(struct nm_generic_mit *mit) 501{ 502 503 return 0; 504} 505 506 507void 508nm_os_mitigation_cleanup(struct nm_generic_mit *mit) 509{ 510} 511 512static int 513nm_vi_dummy(if_t ifp, u_long cmd, caddr_t addr) 514{ 515 516 return EINVAL; 517} 518 519static void 520nm_vi_start(if_t ifp) 521{ 522 panic("nm_vi_start() must not be called"); 523} 524 525/* 526 * Index manager of persistent virtual interfaces. 527 * It is used to decide the lowest byte of the MAC address. 528 * We use the same algorithm with management of bridge port index. 529 */ 530#define NM_VI_MAX 255 531static struct { 532 uint8_t index[NM_VI_MAX]; /* XXX just for a reasonable number */ 533 uint8_t active; 534 struct mtx lock; 535} nm_vi_indices; 536 537void 538nm_os_vi_init_index(void) 539{ 540 int i; 541 for (i = 0; i < NM_VI_MAX; i++) 542 nm_vi_indices.index[i] = i; 543 nm_vi_indices.active = 0; 544 mtx_init(&nm_vi_indices.lock, "nm_vi_indices_lock", NULL, MTX_DEF); 545} 546 547/* return -1 if no index available */ 548static int 549nm_vi_get_index(void) 550{ 551 int ret; 552 553 mtx_lock(&nm_vi_indices.lock); 554 ret = nm_vi_indices.active == NM_VI_MAX ? -1 : 555 nm_vi_indices.index[nm_vi_indices.active++]; 556 mtx_unlock(&nm_vi_indices.lock); 557 return ret; 558} 559 560static void 561nm_vi_free_index(uint8_t val) 562{ 563 int i, lim; 564 565 mtx_lock(&nm_vi_indices.lock); 566 lim = nm_vi_indices.active; 567 for (i = 0; i < lim; i++) { 568 if (nm_vi_indices.index[i] == val) { 569 /* swap index[lim-1] and j */ 570 int tmp = nm_vi_indices.index[lim-1]; 571 nm_vi_indices.index[lim-1] = val; 572 nm_vi_indices.index[i] = tmp; 573 nm_vi_indices.active--; 574 break; 575 } 576 } 577 if (lim == nm_vi_indices.active) 578 nm_prerr("Index %u not found", val); 579 mtx_unlock(&nm_vi_indices.lock); 580} 581#undef NM_VI_MAX 582 583/* 584 * Implementation of a netmap-capable virtual interface that 585 * registered to the system. 586 * It is based on if_tap.c and ip_fw_log.c in FreeBSD 9. 587 * 588 * Note: Linux sets refcount to 0 on allocation of net_device, 589 * then increments it on registration to the system. 590 * FreeBSD sets refcount to 1 on if_alloc(), and does not 591 * increment this refcount on if_attach(). 592 */ 593int 594nm_os_vi_persist(const char *name, if_t *ret) 595{ 596 if_t ifp; 597 u_short macaddr_hi; 598 uint32_t macaddr_mid; 599 u_char eaddr[6]; 600 int unit = nm_vi_get_index(); /* just to decide MAC address */ 601 602 if (unit < 0) 603 return EBUSY; 604 /* 605 * We use the same MAC address generation method with tap 606 * except for the highest octet is 00:be instead of 00:bd 607 */ 608 macaddr_hi = htons(0x00be); /* XXX tap + 1 */ 609 macaddr_mid = (uint32_t) ticks; 610 bcopy(&macaddr_hi, eaddr, sizeof(short)); 611 bcopy(&macaddr_mid, &eaddr[2], sizeof(uint32_t)); 612 eaddr[5] = (uint8_t)unit; 613 614 ifp = if_alloc(IFT_ETHER); 615 if (ifp == NULL) { 616 nm_prerr("if_alloc failed"); 617 return ENOMEM; 618 } 619 if_initname(ifp, name, IF_DUNIT_NONE); 620 if_setflags(ifp, IFF_UP | IFF_SIMPLEX | IFF_MULTICAST); 621 if_setinitfn(ifp, (void *)nm_vi_dummy); 622 if_setioctlfn(ifp, nm_vi_dummy); 623 if_setstartfn(ifp, nm_vi_start); 624 if_setmtu(ifp, ETHERMTU); 625 if_setsendqlen(ifp, ifqmaxlen); 626 if_setcapabilitiesbit(ifp, IFCAP_LINKSTATE, 0); 627 if_setcapenablebit(ifp, IFCAP_LINKSTATE, 0); 628 629 ether_ifattach(ifp, eaddr); 630 *ret = ifp; 631 return 0; 632} 633 634/* unregister from the system and drop the final refcount */ 635void 636nm_os_vi_detach(if_t ifp) 637{ 638 nm_vi_free_index(((char *)if_getlladdr(ifp))[5]); 639 ether_ifdetach(ifp); 640 if_free(ifp); 641} 642 643#ifdef WITH_EXTMEM 644#include <vm/vm_map.h> 645#include <vm/vm_extern.h> 646#include <vm/vm_kern.h> 647struct nm_os_extmem { 648 vm_object_t obj; 649 vm_offset_t kva; 650 vm_offset_t size; 651 uintptr_t scan; 652}; 653 654void 655nm_os_extmem_delete(struct nm_os_extmem *e) 656{ 657 nm_prinf("freeing %zx bytes", (size_t)e->size); 658 vm_map_remove(kernel_map, e->kva, e->kva + e->size); 659 nm_os_free(e); 660} 661 662char * 663nm_os_extmem_nextpage(struct nm_os_extmem *e) 664{ 665 char *rv = NULL; 666 if (e->scan < e->kva + e->size) { 667 rv = (char *)e->scan; 668 e->scan += PAGE_SIZE; 669 } 670 return rv; 671} 672 673int 674nm_os_extmem_isequal(struct nm_os_extmem *e1, struct nm_os_extmem *e2) 675{ 676 return (e1->obj == e2->obj); 677} 678 679int 680nm_os_extmem_nr_pages(struct nm_os_extmem *e) 681{ 682 return e->size >> PAGE_SHIFT; 683} 684 685struct nm_os_extmem * 686nm_os_extmem_create(unsigned long p, struct nmreq_pools_info *pi, int *perror) 687{ 688 vm_map_t map; 689 vm_map_entry_t entry; 690 vm_object_t obj; 691 vm_prot_t prot; 692 vm_pindex_t index; 693 boolean_t wired; 694 struct nm_os_extmem *e = NULL; 695 int rv, error = 0; 696 697 e = nm_os_malloc(sizeof(*e)); 698 if (e == NULL) { 699 error = ENOMEM; 700 goto out; 701 } 702 703 map = &curthread->td_proc->p_vmspace->vm_map; 704 rv = vm_map_lookup(&map, p, VM_PROT_RW, &entry, 705 &obj, &index, &prot, &wired); 706 if (rv != KERN_SUCCESS) { 707 nm_prerr("address %lx not found", p); 708 error = vm_mmap_to_errno(rv); 709 goto out_free; 710 } 711 vm_object_reference(obj); 712 713 /* check that we are given the whole vm_object ? */ 714 vm_map_lookup_done(map, entry); 715 716 e->obj = obj; 717 /* Wire the memory and add the vm_object to the kernel map, 718 * to make sure that it is not freed even if all the processes 719 * that are mmap()ing should munmap() it. 720 */ 721 e->kva = vm_map_min(kernel_map); 722 e->size = obj->size << PAGE_SHIFT; 723 rv = vm_map_find(kernel_map, obj, 0, &e->kva, e->size, 0, 724 VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 725 VM_PROT_READ | VM_PROT_WRITE, 0); 726 if (rv != KERN_SUCCESS) { 727 nm_prerr("vm_map_find(%zx) failed", (size_t)e->size); 728 error = vm_mmap_to_errno(rv); 729 goto out_rel; 730 } 731 rv = vm_map_wire(kernel_map, e->kva, e->kva + e->size, 732 VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); 733 if (rv != KERN_SUCCESS) { 734 nm_prerr("vm_map_wire failed"); 735 error = vm_mmap_to_errno(rv); 736 goto out_rem; 737 } 738 739 e->scan = e->kva; 740 741 return e; 742 743out_rem: 744 vm_map_remove(kernel_map, e->kva, e->kva + e->size); 745out_rel: 746 vm_object_deallocate(e->obj); 747 e->obj = NULL; 748out_free: 749 nm_os_free(e); 750out: 751 if (perror) 752 *perror = error; 753 return NULL; 754} 755#endif /* WITH_EXTMEM */ 756 757/* ================== PTNETMAP GUEST SUPPORT ==================== */ 758 759#ifdef WITH_PTNETMAP 760#include <sys/bus.h> 761#include <sys/rman.h> 762#include <machine/bus.h> /* bus_dmamap_* */ 763#include <machine/resource.h> 764#include <dev/pci/pcivar.h> 765#include <dev/pci/pcireg.h> 766/* 767 * ptnetmap memory device (memdev) for freebsd guest, 768 * ssed to expose host netmap memory to the guest through a PCI BAR. 769 */ 770 771/* 772 * ptnetmap memdev private data structure 773 */ 774struct ptnetmap_memdev { 775 device_t dev; 776 struct resource *pci_io; 777 struct resource *pci_mem; 778 struct netmap_mem_d *nm_mem; 779}; 780 781static int ptn_memdev_probe(device_t); 782static int ptn_memdev_attach(device_t); 783static int ptn_memdev_detach(device_t); 784static int ptn_memdev_shutdown(device_t); 785 786static device_method_t ptn_memdev_methods[] = { 787 DEVMETHOD(device_probe, ptn_memdev_probe), 788 DEVMETHOD(device_attach, ptn_memdev_attach), 789 DEVMETHOD(device_detach, ptn_memdev_detach), 790 DEVMETHOD(device_shutdown, ptn_memdev_shutdown), 791 DEVMETHOD_END 792}; 793 794static driver_t ptn_memdev_driver = { 795 PTNETMAP_MEMDEV_NAME, 796 ptn_memdev_methods, 797 sizeof(struct ptnetmap_memdev), 798}; 799 800/* We use (SI_ORDER_MIDDLE+1) here, see DEV_MODULE_ORDERED() invocation 801 * below. */ 802DRIVER_MODULE_ORDERED(ptn_memdev, pci, ptn_memdev_driver, NULL, NULL, 803 SI_ORDER_MIDDLE + 1); 804 805/* 806 * Map host netmap memory through PCI-BAR in the guest OS, 807 * returning physical (nm_paddr) and virtual (nm_addr) addresses 808 * of the netmap memory mapped in the guest. 809 */ 810int 811nm_os_pt_memdev_iomap(struct ptnetmap_memdev *ptn_dev, vm_paddr_t *nm_paddr, 812 void **nm_addr, uint64_t *mem_size) 813{ 814 int rid; 815 816 nm_prinf("ptn_memdev_driver iomap"); 817 818 rid = PCIR_BAR(PTNETMAP_MEM_PCI_BAR); 819 *mem_size = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMSIZE_HI); 820 *mem_size = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMSIZE_LO) | 821 (*mem_size << 32); 822 823 /* map memory allocator */ 824 ptn_dev->pci_mem = bus_alloc_resource(ptn_dev->dev, SYS_RES_MEMORY, 825 &rid, 0, ~0, *mem_size, RF_ACTIVE); 826 if (ptn_dev->pci_mem == NULL) { 827 *nm_paddr = 0; 828 *nm_addr = NULL; 829 return ENOMEM; 830 } 831 832 *nm_paddr = rman_get_start(ptn_dev->pci_mem); 833 *nm_addr = rman_get_virtual(ptn_dev->pci_mem); 834 835 nm_prinf("=== BAR %d start %lx len %lx mem_size %lx ===", 836 PTNETMAP_MEM_PCI_BAR, 837 (unsigned long)(*nm_paddr), 838 (unsigned long)rman_get_size(ptn_dev->pci_mem), 839 (unsigned long)*mem_size); 840 return (0); 841} 842 843uint32_t 844nm_os_pt_memdev_ioread(struct ptnetmap_memdev *ptn_dev, unsigned int reg) 845{ 846 return bus_read_4(ptn_dev->pci_io, reg); 847} 848 849/* Unmap host netmap memory. */ 850void 851nm_os_pt_memdev_iounmap(struct ptnetmap_memdev *ptn_dev) 852{ 853 nm_prinf("ptn_memdev_driver iounmap"); 854 855 if (ptn_dev->pci_mem) { 856 bus_release_resource(ptn_dev->dev, SYS_RES_MEMORY, 857 PCIR_BAR(PTNETMAP_MEM_PCI_BAR), ptn_dev->pci_mem); 858 ptn_dev->pci_mem = NULL; 859 } 860} 861 862/* Device identification routine, return BUS_PROBE_DEFAULT on success, 863 * positive on failure */ 864static int 865ptn_memdev_probe(device_t dev) 866{ 867 char desc[256]; 868 869 if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID) 870 return (ENXIO); 871 if (pci_get_device(dev) != PTNETMAP_PCI_DEVICE_ID) 872 return (ENXIO); 873 874 snprintf(desc, sizeof(desc), "%s PCI adapter", 875 PTNETMAP_MEMDEV_NAME); 876 device_set_desc_copy(dev, desc); 877 878 return (BUS_PROBE_DEFAULT); 879} 880 881/* Device initialization routine. */ 882static int 883ptn_memdev_attach(device_t dev) 884{ 885 struct ptnetmap_memdev *ptn_dev; 886 int rid; 887 uint16_t mem_id; 888 889 ptn_dev = device_get_softc(dev); 890 ptn_dev->dev = dev; 891 892 pci_enable_busmaster(dev); 893 894 rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR); 895 ptn_dev->pci_io = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 896 RF_ACTIVE); 897 if (ptn_dev->pci_io == NULL) { 898 device_printf(dev, "cannot map I/O space\n"); 899 return (ENXIO); 900 } 901 902 mem_id = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMID); 903 904 /* create guest allocator */ 905 ptn_dev->nm_mem = netmap_mem_pt_guest_attach(ptn_dev, mem_id); 906 if (ptn_dev->nm_mem == NULL) { 907 ptn_memdev_detach(dev); 908 return (ENOMEM); 909 } 910 netmap_mem_get(ptn_dev->nm_mem); 911 912 nm_prinf("ptnetmap memdev attached, host memid: %u", mem_id); 913 914 return (0); 915} 916 917/* Device removal routine. */ 918static int 919ptn_memdev_detach(device_t dev) 920{ 921 struct ptnetmap_memdev *ptn_dev; 922 923 ptn_dev = device_get_softc(dev); 924 925 if (ptn_dev->nm_mem) { 926 nm_prinf("ptnetmap memdev detached, host memid %u", 927 netmap_mem_get_id(ptn_dev->nm_mem)); 928 netmap_mem_put(ptn_dev->nm_mem); 929 ptn_dev->nm_mem = NULL; 930 } 931 if (ptn_dev->pci_mem) { 932 bus_release_resource(dev, SYS_RES_MEMORY, 933 PCIR_BAR(PTNETMAP_MEM_PCI_BAR), ptn_dev->pci_mem); 934 ptn_dev->pci_mem = NULL; 935 } 936 if (ptn_dev->pci_io) { 937 bus_release_resource(dev, SYS_RES_IOPORT, 938 PCIR_BAR(PTNETMAP_IO_PCI_BAR), ptn_dev->pci_io); 939 ptn_dev->pci_io = NULL; 940 } 941 942 return (0); 943} 944 945static int 946ptn_memdev_shutdown(device_t dev) 947{ 948 return bus_generic_shutdown(dev); 949} 950 951#endif /* WITH_PTNETMAP */ 952 953/* 954 * In order to track whether pages are still mapped, we hook into 955 * the standard cdev_pager and intercept the constructor and 956 * destructor. 957 */ 958 959struct netmap_vm_handle_t { 960 struct cdev *dev; 961 struct netmap_priv_d *priv; 962}; 963 964 965static int 966netmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 967 vm_ooffset_t foff, struct ucred *cred, u_short *color) 968{ 969 struct netmap_vm_handle_t *vmh = handle; 970 971 if (netmap_verbose) 972 nm_prinf("handle %p size %jd prot %d foff %jd", 973 handle, (intmax_t)size, prot, (intmax_t)foff); 974 if (color) 975 *color = 0; 976 dev_ref(vmh->dev); 977 return 0; 978} 979 980 981static void 982netmap_dev_pager_dtor(void *handle) 983{ 984 struct netmap_vm_handle_t *vmh = handle; 985 struct cdev *dev = vmh->dev; 986 struct netmap_priv_d *priv = vmh->priv; 987 988 if (netmap_verbose) 989 nm_prinf("handle %p", handle); 990 netmap_dtor(priv); 991 free(vmh, M_DEVBUF); 992 dev_rel(dev); 993} 994 995 996static int 997netmap_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, 998 int prot, vm_page_t *mres) 999{ 1000 struct netmap_vm_handle_t *vmh = object->handle; 1001 struct netmap_priv_d *priv = vmh->priv; 1002 struct netmap_adapter *na = priv->np_na; 1003 vm_paddr_t paddr; 1004 vm_page_t page; 1005 vm_memattr_t memattr; 1006 1007 nm_prdis("object %p offset %jd prot %d mres %p", 1008 object, (intmax_t)offset, prot, mres); 1009 memattr = object->memattr; 1010 paddr = netmap_mem_ofstophys(na->nm_mem, offset); 1011 if (paddr == 0) 1012 return VM_PAGER_FAIL; 1013 1014 if (((*mres)->flags & PG_FICTITIOUS) != 0) { 1015 /* 1016 * If the passed in result page is a fake page, update it with 1017 * the new physical address. 1018 */ 1019 page = *mres; 1020 vm_page_updatefake(page, paddr, memattr); 1021 } else { 1022 /* 1023 * Replace the passed in reqpage page with our own fake page and 1024 * free up the all of the original pages. 1025 */ 1026 VM_OBJECT_WUNLOCK(object); 1027 page = vm_page_getfake(paddr, memattr); 1028 VM_OBJECT_WLOCK(object); 1029 vm_page_replace(page, object, (*mres)->pindex, *mres); 1030 *mres = page; 1031 } 1032 page->valid = VM_PAGE_BITS_ALL; 1033 return (VM_PAGER_OK); 1034} 1035 1036 1037static struct cdev_pager_ops netmap_cdev_pager_ops = { 1038 .cdev_pg_ctor = netmap_dev_pager_ctor, 1039 .cdev_pg_dtor = netmap_dev_pager_dtor, 1040 .cdev_pg_fault = netmap_dev_pager_fault, 1041}; 1042 1043 1044static int 1045netmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff, 1046 vm_size_t objsize, vm_object_t *objp, int prot) 1047{ 1048 int error; 1049 struct netmap_vm_handle_t *vmh; 1050 struct netmap_priv_d *priv; 1051 vm_object_t obj; 1052 1053 if (netmap_verbose) 1054 nm_prinf("cdev %p foff %jd size %jd objp %p prot %d", cdev, 1055 (intmax_t )*foff, (intmax_t )objsize, objp, prot); 1056 1057 vmh = malloc(sizeof(struct netmap_vm_handle_t), M_DEVBUF, 1058 M_NOWAIT | M_ZERO); 1059 if (vmh == NULL) 1060 return ENOMEM; 1061 vmh->dev = cdev; 1062 1063 NMG_LOCK(); 1064 error = devfs_get_cdevpriv((void**)&priv); 1065 if (error) 1066 goto err_unlock; 1067 if (priv->np_nifp == NULL) { 1068 error = EINVAL; 1069 goto err_unlock; 1070 } 1071 vmh->priv = priv; 1072 priv->np_refs++; 1073 NMG_UNLOCK(); 1074 1075 obj = cdev_pager_allocate(vmh, OBJT_DEVICE, 1076 &netmap_cdev_pager_ops, objsize, prot, 1077 *foff, NULL); 1078 if (obj == NULL) { 1079 nm_prerr("cdev_pager_allocate failed"); 1080 error = EINVAL; 1081 goto err_deref; 1082 } 1083 1084 *objp = obj; 1085 return 0; 1086 1087err_deref: 1088 NMG_LOCK(); 1089 priv->np_refs--; 1090err_unlock: 1091 NMG_UNLOCK(); 1092// err: 1093 free(vmh, M_DEVBUF); 1094 return error; 1095} 1096 1097/* 1098 * On FreeBSD the close routine is only called on the last close on 1099 * the device (/dev/netmap) so we cannot do anything useful. 1100 * To track close() on individual file descriptors we pass netmap_dtor() to 1101 * devfs_set_cdevpriv() on open(). The FreeBSD kernel will call the destructor 1102 * when the last fd pointing to the device is closed. 1103 * 1104 * Note that FreeBSD does not even munmap() on close() so we also have 1105 * to track mmap() ourselves, and postpone the call to 1106 * netmap_dtor() is called when the process has no open fds and no active 1107 * memory maps on /dev/netmap, as in linux. 1108 */ 1109static int 1110netmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 1111{ 1112 if (netmap_verbose) 1113 nm_prinf("dev %p fflag 0x%x devtype %d td %p", 1114 dev, fflag, devtype, td); 1115 return 0; 1116} 1117 1118 1119static int 1120netmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 1121{ 1122 struct netmap_priv_d *priv; 1123 int error; 1124 1125 (void)dev; 1126 (void)oflags; 1127 (void)devtype; 1128 (void)td; 1129 1130 NMG_LOCK(); 1131 priv = netmap_priv_new(); 1132 if (priv == NULL) { 1133 error = ENOMEM; 1134 goto out; 1135 } 1136 error = devfs_set_cdevpriv(priv, netmap_dtor); 1137 if (error) { 1138 netmap_priv_delete(priv); 1139 } 1140out: 1141 NMG_UNLOCK(); 1142 return error; 1143} 1144 1145/******************** kthread wrapper ****************/ 1146#include <sys/sysproto.h> 1147u_int 1148nm_os_ncpus(void) 1149{ 1150 return mp_maxid + 1; 1151} 1152 1153struct nm_kctx_ctx { 1154 /* Userspace thread (kthread creator). */ 1155 struct thread *user_td; 1156 1157 /* worker function and parameter */ 1158 nm_kctx_worker_fn_t worker_fn; 1159 void *worker_private; 1160 1161 struct nm_kctx *nmk; 1162 1163 /* integer to manage multiple worker contexts (e.g., RX or TX on ptnetmap) */ 1164 long type; 1165}; 1166 1167struct nm_kctx { 1168 struct thread *worker; 1169 struct mtx worker_lock; 1170 struct nm_kctx_ctx worker_ctx; 1171 int run; /* used to stop kthread */ 1172 int attach_user; /* kthread attached to user_process */ 1173 int affinity; 1174}; 1175 1176static void 1177nm_kctx_worker(void *data) 1178{ 1179 struct nm_kctx *nmk = data; 1180 struct nm_kctx_ctx *ctx = &nmk->worker_ctx; 1181 1182 if (nmk->affinity >= 0) { 1183 thread_lock(curthread); 1184 sched_bind(curthread, nmk->affinity); 1185 thread_unlock(curthread); 1186 } 1187 1188 while (nmk->run) { 1189 /* 1190 * check if the parent process dies 1191 * (when kthread is attached to user process) 1192 */ 1193 if (ctx->user_td) { 1194 PROC_LOCK(curproc); 1195 thread_suspend_check(0); 1196 PROC_UNLOCK(curproc); 1197 } else { 1198 kthread_suspend_check(); 1199 } 1200 1201 /* Continuously execute worker process. */ 1202 ctx->worker_fn(ctx->worker_private); /* worker body */ 1203 } 1204 1205 kthread_exit(); 1206} 1207 1208void 1209nm_os_kctx_worker_setaff(struct nm_kctx *nmk, int affinity) 1210{ 1211 nmk->affinity = affinity; 1212} 1213 1214struct nm_kctx * 1215nm_os_kctx_create(struct nm_kctx_cfg *cfg, void *opaque) 1216{ 1217 struct nm_kctx *nmk = NULL; 1218 1219 nmk = malloc(sizeof(*nmk), M_DEVBUF, M_NOWAIT | M_ZERO); 1220 if (!nmk) 1221 return NULL; 1222 1223 mtx_init(&nmk->worker_lock, "nm_kthread lock", NULL, MTX_DEF); 1224 nmk->worker_ctx.worker_fn = cfg->worker_fn; 1225 nmk->worker_ctx.worker_private = cfg->worker_private; 1226 nmk->worker_ctx.type = cfg->type; 1227 nmk->affinity = -1; 1228 1229 /* attach kthread to user process (ptnetmap) */ 1230 nmk->attach_user = cfg->attach_user; 1231 1232 return nmk; 1233} 1234 1235int 1236nm_os_kctx_worker_start(struct nm_kctx *nmk) 1237{ 1238 struct proc *p = NULL; 1239 int error = 0; 1240 1241 /* Temporarily disable this function as it is currently broken 1242 * and causes kernel crashes. The failure can be triggered by 1243 * the "vale_polling_enable_disable" test in ctrl-api-test.c. */ 1244 return EOPNOTSUPP; 1245 1246 if (nmk->worker) 1247 return EBUSY; 1248 1249 /* check if we want to attach kthread to user process */ 1250 if (nmk->attach_user) { 1251 nmk->worker_ctx.user_td = curthread; 1252 p = curthread->td_proc; 1253 } 1254 1255 /* enable kthread main loop */ 1256 nmk->run = 1; 1257 /* create kthread */ 1258 if((error = kthread_add(nm_kctx_worker, nmk, p, 1259 &nmk->worker, RFNOWAIT /* to be checked */, 0, "nm-kthread-%ld", 1260 nmk->worker_ctx.type))) { 1261 goto err; 1262 } 1263 1264 nm_prinf("nm_kthread started td %p", nmk->worker); 1265 1266 return 0; 1267err: 1268 nm_prerr("nm_kthread start failed err %d", error); 1269 nmk->worker = NULL; 1270 return error; 1271} 1272 1273void 1274nm_os_kctx_worker_stop(struct nm_kctx *nmk) 1275{ 1276 if (!nmk->worker) 1277 return; 1278 1279 /* tell to kthread to exit from main loop */ 1280 nmk->run = 0; 1281 1282 /* wake up kthread if it sleeps */ 1283 kthread_resume(nmk->worker); 1284 1285 nmk->worker = NULL; 1286} 1287 1288void 1289nm_os_kctx_destroy(struct nm_kctx *nmk) 1290{ 1291 if (!nmk) 1292 return; 1293 1294 if (nmk->worker) 1295 nm_os_kctx_worker_stop(nmk); 1296 1297 free(nmk, M_DEVBUF); 1298} 1299 1300/******************** kqueue support ****************/ 1301 1302/* 1303 * In addition to calling selwakeuppri(), nm_os_selwakeup() also 1304 * needs to call knote() to wake up kqueue listeners. 1305 * This operation is deferred to a taskqueue in order to avoid possible 1306 * lock order reversals; these may happen because knote() grabs a 1307 * private lock associated to the 'si' (see struct selinfo, 1308 * struct nm_selinfo, and nm_os_selinfo_init), and nm_os_selwakeup() 1309 * can be called while holding the lock associated to a different 1310 * 'si'. 1311 * When calling knote() we use a non-zero 'hint' argument to inform 1312 * the netmap_knrw() function that it is being called from 1313 * 'nm_os_selwakeup'; this is necessary because when netmap_knrw() is 1314 * called by the kevent subsystem (i.e. kevent_scan()) we also need to 1315 * call netmap_poll(). 1316 * 1317 * The netmap_kqfilter() function registers one or another f_event 1318 * depending on read or write mode. A pointer to the struct 1319 * 'netmap_priv_d' is stored into kn->kn_hook, so that it can later 1320 * be passed to netmap_poll(). We pass NULL as a third argument to 1321 * netmap_poll(), so that the latter only runs the txsync/rxsync 1322 * (if necessary), and skips the nm_os_selrecord() calls. 1323 */ 1324 1325 1326void 1327nm_os_selwakeup(struct nm_selinfo *si) 1328{ 1329 selwakeuppri(&si->si, PI_NET); 1330 if (si->kqueue_users > 0) { 1331 taskqueue_enqueue(si->ntfytq, &si->ntfytask); 1332 } 1333} 1334 1335void 1336nm_os_selrecord(struct thread *td, struct nm_selinfo *si) 1337{ 1338 selrecord(td, &si->si); 1339} 1340 1341static void 1342netmap_knrdetach(struct knote *kn) 1343{ 1344 struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; 1345 struct nm_selinfo *si = priv->np_si[NR_RX]; 1346 1347 knlist_remove(&si->si.si_note, kn, /*islocked=*/0); 1348 NMG_LOCK(); 1349 KASSERT(si->kqueue_users > 0, ("kqueue_user underflow on %s", 1350 si->mtxname)); 1351 si->kqueue_users--; 1352 nm_prinf("kqueue users for %s: %d", si->mtxname, si->kqueue_users); 1353 NMG_UNLOCK(); 1354} 1355 1356static void 1357netmap_knwdetach(struct knote *kn) 1358{ 1359 struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; 1360 struct nm_selinfo *si = priv->np_si[NR_TX]; 1361 1362 knlist_remove(&si->si.si_note, kn, /*islocked=*/0); 1363 NMG_LOCK(); 1364 si->kqueue_users--; 1365 nm_prinf("kqueue users for %s: %d", si->mtxname, si->kqueue_users); 1366 NMG_UNLOCK(); 1367} 1368 1369/* 1370 * Callback triggered by netmap notifications (see netmap_notify()), 1371 * and by the application calling kevent(). In the former case we 1372 * just return 1 (events ready), since we are not able to do better. 1373 * In the latter case we use netmap_poll() to see which events are 1374 * ready. 1375 */ 1376static int 1377netmap_knrw(struct knote *kn, long hint, int events) 1378{ 1379 struct netmap_priv_d *priv; 1380 int revents; 1381 1382 if (hint != 0) { 1383 /* Called from netmap_notify(), typically from a 1384 * thread different from the one issuing kevent(). 1385 * Assume we are ready. */ 1386 return 1; 1387 } 1388 1389 /* Called from kevent(). */ 1390 priv = kn->kn_hook; 1391 revents = netmap_poll(priv, events, /*thread=*/NULL); 1392 1393 return (events & revents) ? 1 : 0; 1394} 1395 1396static int 1397netmap_knread(struct knote *kn, long hint) 1398{ 1399 return netmap_knrw(kn, hint, POLLIN); 1400} 1401 1402static int 1403netmap_knwrite(struct knote *kn, long hint) 1404{ 1405 return netmap_knrw(kn, hint, POLLOUT); 1406} 1407 1408static struct filterops netmap_rfiltops = { 1409 .f_isfd = 1, 1410 .f_detach = netmap_knrdetach, 1411 .f_event = netmap_knread, 1412}; 1413 1414static struct filterops netmap_wfiltops = { 1415 .f_isfd = 1, 1416 .f_detach = netmap_knwdetach, 1417 .f_event = netmap_knwrite, 1418}; 1419 1420 1421/* 1422 * This is called when a thread invokes kevent() to record 1423 * a change in the configuration of the kqueue(). 1424 * The 'priv' is the one associated to the open netmap device. 1425 */ 1426static int 1427netmap_kqfilter(struct cdev *dev, struct knote *kn) 1428{ 1429 struct netmap_priv_d *priv; 1430 int error; 1431 struct netmap_adapter *na; 1432 struct nm_selinfo *si; 1433 int ev = kn->kn_filter; 1434 1435 if (ev != EVFILT_READ && ev != EVFILT_WRITE) { 1436 nm_prerr("bad filter request %d", ev); 1437 return 1; 1438 } 1439 error = devfs_get_cdevpriv((void**)&priv); 1440 if (error) { 1441 nm_prerr("device not yet setup"); 1442 return 1; 1443 } 1444 na = priv->np_na; 1445 if (na == NULL) { 1446 nm_prerr("no netmap adapter for this file descriptor"); 1447 return 1; 1448 } 1449 /* the si is indicated in the priv */ 1450 si = priv->np_si[(ev == EVFILT_WRITE) ? NR_TX : NR_RX]; 1451 kn->kn_fop = (ev == EVFILT_WRITE) ? 1452 &netmap_wfiltops : &netmap_rfiltops; 1453 kn->kn_hook = priv; 1454 NMG_LOCK(); 1455 si->kqueue_users++; 1456 nm_prinf("kqueue users for %s: %d", si->mtxname, si->kqueue_users); 1457 NMG_UNLOCK(); 1458 knlist_add(&si->si.si_note, kn, /*islocked=*/0); 1459 1460 return 0; 1461} 1462 1463static int 1464freebsd_netmap_poll(struct cdev *cdevi __unused, int events, struct thread *td) 1465{ 1466 struct netmap_priv_d *priv; 1467 if (devfs_get_cdevpriv((void **)&priv)) { 1468 return POLLERR; 1469 } 1470 return netmap_poll(priv, events, td); 1471} 1472 1473static int 1474freebsd_netmap_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t data, 1475 int ffla __unused, struct thread *td) 1476{ 1477 int error; 1478 struct netmap_priv_d *priv; 1479 1480 CURVNET_SET(TD_TO_VNET(td)); 1481 error = devfs_get_cdevpriv((void **)&priv); 1482 if (error) { 1483 /* XXX ENOENT should be impossible, since the priv 1484 * is now created in the open */ 1485 if (error == ENOENT) 1486 error = ENXIO; 1487 goto out; 1488 } 1489 error = netmap_ioctl(priv, cmd, data, td, /*nr_body_is_user=*/1); 1490out: 1491 CURVNET_RESTORE(); 1492 1493 return error; 1494} 1495 1496void 1497nm_os_onattach(if_t ifp) 1498{ 1499 if_setcapabilitiesbit(ifp, IFCAP_NETMAP, 0); 1500} 1501 1502void 1503nm_os_onenter(if_t ifp) 1504{ 1505 struct netmap_adapter *na = NA(ifp); 1506 1507 na->if_transmit = if_gettransmitfn(ifp); 1508 if_settransmitfn(ifp, netmap_transmit); 1509 if_setcapenablebit(ifp, IFCAP_NETMAP, 0); 1510} 1511 1512void 1513nm_os_onexit(if_t ifp) 1514{ 1515 struct netmap_adapter *na = NA(ifp); 1516 1517 if_settransmitfn(ifp, na->if_transmit); 1518 if_setcapenablebit(ifp, 0, IFCAP_NETMAP); 1519} 1520 1521extern struct cdevsw netmap_cdevsw; /* XXX used in netmap.c, should go elsewhere */ 1522struct cdevsw netmap_cdevsw = { 1523 .d_version = D_VERSION, 1524 .d_name = "netmap", 1525 .d_open = netmap_open, 1526 .d_mmap_single = netmap_mmap_single, 1527 .d_ioctl = freebsd_netmap_ioctl, 1528 .d_poll = freebsd_netmap_poll, 1529 .d_kqfilter = netmap_kqfilter, 1530 .d_close = netmap_close, 1531}; 1532/*--- end of kqueue support ----*/ 1533 1534/* 1535 * Kernel entry point. 1536 * 1537 * Initialize/finalize the module and return. 1538 * 1539 * Return 0 on success, errno on failure. 1540 */ 1541static int 1542netmap_loader(__unused struct module *module, int event, __unused void *arg) 1543{ 1544 int error = 0; 1545 1546 switch (event) { 1547 case MOD_LOAD: 1548 error = netmap_init(); 1549 break; 1550 1551 case MOD_UNLOAD: 1552 /* 1553 * if some one is still using netmap, 1554 * then the module can not be unloaded. 1555 */ 1556 if (netmap_use_count) { 1557 nm_prerr("netmap module can not be unloaded - netmap_use_count: %d", 1558 netmap_use_count); 1559 error = EBUSY; 1560 break; 1561 } 1562 netmap_fini(); 1563 break; 1564 1565 default: 1566 error = EOPNOTSUPP; 1567 break; 1568 } 1569 1570 return (error); 1571} 1572 1573#ifdef DEV_MODULE_ORDERED 1574/* 1575 * The netmap module contains three drivers: (i) the netmap character device 1576 * driver; (ii) the ptnetmap memdev PCI device driver, (iii) the ptnet PCI 1577 * device driver. The attach() routines of both (ii) and (iii) need the 1578 * lock of the global allocator, and such lock is initialized in netmap_init(), 1579 * which is part of (i). 1580 * Therefore, we make sure that (i) is loaded before (ii) and (iii), using 1581 * the 'order' parameter of driver declaration macros. For (i), we specify 1582 * SI_ORDER_MIDDLE, while higher orders are used with the DRIVER_MODULE_ORDERED 1583 * macros for (ii) and (iii). 1584 */ 1585DEV_MODULE_ORDERED(netmap, netmap_loader, NULL, SI_ORDER_MIDDLE); 1586#else /* !DEV_MODULE_ORDERED */ 1587DEV_MODULE(netmap, netmap_loader, NULL); 1588#endif /* DEV_MODULE_ORDERED */ 1589MODULE_DEPEND(netmap, pci, 1, 1, 1); 1590MODULE_VERSION(netmap, 1); 1591/* reduce conditional code */ 1592// linux API, use for the knlist in FreeBSD 1593/* use a private mutex for the knlist */ 1594