bpf.c revision 227309
1/*- 2 * Copyright (c) 1990, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from the Stanford/CMU enet packet filter, 6 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 7 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 8 * Berkeley Laboratory. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 35 */ 36 37#include <sys/cdefs.h> 38__FBSDID("$FreeBSD: head/sys/net/bpf.c 227309 2011-11-07 15:43:11Z ed $"); 39 40#include "opt_bpf.h" 41#include "opt_compat.h" 42#include "opt_netgraph.h" 43 44#include <sys/types.h> 45#include <sys/param.h> 46#include <sys/systm.h> 47#include <sys/conf.h> 48#include <sys/fcntl.h> 49#include <sys/jail.h> 50#include <sys/malloc.h> 51#include <sys/mbuf.h> 52#include <sys/time.h> 53#include <sys/priv.h> 54#include <sys/proc.h> 55#include <sys/signalvar.h> 56#include <sys/filio.h> 57#include <sys/sockio.h> 58#include <sys/ttycom.h> 59#include <sys/uio.h> 60 61#include <sys/event.h> 62#include <sys/file.h> 63#include <sys/poll.h> 64#include <sys/proc.h> 65 66#include <sys/socket.h> 67 68#include <net/if.h> 69#include <net/bpf.h> 70#include <net/bpf_buffer.h> 71#ifdef BPF_JITTER 72#include <net/bpf_jitter.h> 73#endif 74#include <net/bpf_zerocopy.h> 75#include <net/bpfdesc.h> 76#include <net/vnet.h> 77 78#include <netinet/in.h> 79#include <netinet/if_ether.h> 80#include <sys/kernel.h> 81#include <sys/sysctl.h> 82 83#include <net80211/ieee80211_freebsd.h> 84 85#include <security/mac/mac_framework.h> 86 87MALLOC_DEFINE(M_BPF, "BPF", "BPF data"); 88 89#if defined(DEV_BPF) || defined(NETGRAPH_BPF) 90 91#define PRINET 26 /* interruptible */ 92 93#define SIZEOF_BPF_HDR(type) \ 94 (offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen)) 95 96#ifdef COMPAT_FREEBSD32 97#include <sys/mount.h> 98#include <compat/freebsd32/freebsd32.h> 99#define BPF_ALIGNMENT32 sizeof(int32_t) 100#define BPF_WORDALIGN32(x) (((x)+(BPF_ALIGNMENT32-1))&~(BPF_ALIGNMENT32-1)) 101 102#ifndef BURN_BRIDGES 103/* 104 * 32-bit version of structure prepended to each packet. We use this header 105 * instead of the standard one for 32-bit streams. We mark the a stream as 106 * 32-bit the first time we see a 32-bit compat ioctl request. 107 */ 108struct bpf_hdr32 { 109 struct timeval32 bh_tstamp; /* time stamp */ 110 uint32_t bh_caplen; /* length of captured portion */ 111 uint32_t bh_datalen; /* original length of packet */ 112 uint16_t bh_hdrlen; /* length of bpf header (this struct 113 plus alignment padding) */ 114}; 115#endif 116 117struct bpf_program32 { 118 u_int bf_len; 119 uint32_t bf_insns; 120}; 121 122struct bpf_dltlist32 { 123 u_int bfl_len; 124 u_int bfl_list; 125}; 126 127#define BIOCSETF32 _IOW('B', 103, struct bpf_program32) 128#define BIOCSRTIMEOUT32 _IOW('B', 109, struct timeval32) 129#define BIOCGRTIMEOUT32 _IOR('B', 110, struct timeval32) 130#define BIOCGDLTLIST32 _IOWR('B', 121, struct bpf_dltlist32) 131#define BIOCSETWF32 _IOW('B', 123, struct bpf_program32) 132#define BIOCSETFNR32 _IOW('B', 130, struct bpf_program32) 133#endif 134 135/* 136 * bpf_iflist is a list of BPF interface structures, each corresponding to a 137 * specific DLT. The same network interface might have several BPF interface 138 * structures registered by different layers in the stack (i.e., 802.11 139 * frames, ethernet frames, etc). 140 */ 141static LIST_HEAD(, bpf_if) bpf_iflist; 142static struct mtx bpf_mtx; /* bpf global lock */ 143static int bpf_bpfd_cnt; 144 145static void bpf_attachd(struct bpf_d *, struct bpf_if *); 146static void bpf_detachd(struct bpf_d *); 147static void bpf_freed(struct bpf_d *); 148static int bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **, 149 struct sockaddr *, int *, struct bpf_insn *); 150static int bpf_setif(struct bpf_d *, struct ifreq *); 151static void bpf_timed_out(void *); 152static __inline void 153 bpf_wakeup(struct bpf_d *); 154static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 155 void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int), 156 struct bintime *); 157static void reset_d(struct bpf_d *); 158static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd); 159static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 160static int bpf_setdlt(struct bpf_d *, u_int); 161static void filt_bpfdetach(struct knote *); 162static int filt_bpfread(struct knote *, long); 163static void bpf_drvinit(void *); 164static int bpf_stats_sysctl(SYSCTL_HANDLER_ARGS); 165 166SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW, 0, "bpf sysctl"); 167int bpf_maxinsns = BPF_MAXINSNS; 168SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW, 169 &bpf_maxinsns, 0, "Maximum bpf program instructions"); 170static int bpf_zerocopy_enable = 0; 171SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW, 172 &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions"); 173static SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_MPSAFE | CTLFLAG_RW, 174 bpf_stats_sysctl, "bpf statistics portal"); 175 176static d_open_t bpfopen; 177static d_read_t bpfread; 178static d_write_t bpfwrite; 179static d_ioctl_t bpfioctl; 180static d_poll_t bpfpoll; 181static d_kqfilter_t bpfkqfilter; 182 183static struct cdevsw bpf_cdevsw = { 184 .d_version = D_VERSION, 185 .d_open = bpfopen, 186 .d_read = bpfread, 187 .d_write = bpfwrite, 188 .d_ioctl = bpfioctl, 189 .d_poll = bpfpoll, 190 .d_name = "bpf", 191 .d_kqfilter = bpfkqfilter, 192}; 193 194static struct filterops bpfread_filtops = { 195 .f_isfd = 1, 196 .f_detach = filt_bpfdetach, 197 .f_event = filt_bpfread, 198}; 199 200/* 201 * Wrapper functions for various buffering methods. If the set of buffer 202 * modes expands, we will probably want to introduce a switch data structure 203 * similar to protosw, et. 204 */ 205static void 206bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, 207 u_int len) 208{ 209 210 BPFD_LOCK_ASSERT(d); 211 212 switch (d->bd_bufmode) { 213 case BPF_BUFMODE_BUFFER: 214 return (bpf_buffer_append_bytes(d, buf, offset, src, len)); 215 216 case BPF_BUFMODE_ZBUF: 217 d->bd_zcopy++; 218 return (bpf_zerocopy_append_bytes(d, buf, offset, src, len)); 219 220 default: 221 panic("bpf_buf_append_bytes"); 222 } 223} 224 225static void 226bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src, 227 u_int len) 228{ 229 230 BPFD_LOCK_ASSERT(d); 231 232 switch (d->bd_bufmode) { 233 case BPF_BUFMODE_BUFFER: 234 return (bpf_buffer_append_mbuf(d, buf, offset, src, len)); 235 236 case BPF_BUFMODE_ZBUF: 237 d->bd_zcopy++; 238 return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len)); 239 240 default: 241 panic("bpf_buf_append_mbuf"); 242 } 243} 244 245/* 246 * This function gets called when the free buffer is re-assigned. 247 */ 248static void 249bpf_buf_reclaimed(struct bpf_d *d) 250{ 251 252 BPFD_LOCK_ASSERT(d); 253 254 switch (d->bd_bufmode) { 255 case BPF_BUFMODE_BUFFER: 256 return; 257 258 case BPF_BUFMODE_ZBUF: 259 bpf_zerocopy_buf_reclaimed(d); 260 return; 261 262 default: 263 panic("bpf_buf_reclaimed"); 264 } 265} 266 267/* 268 * If the buffer mechanism has a way to decide that a held buffer can be made 269 * free, then it is exposed via the bpf_canfreebuf() interface. (1) is 270 * returned if the buffer can be discarded, (0) is returned if it cannot. 271 */ 272static int 273bpf_canfreebuf(struct bpf_d *d) 274{ 275 276 BPFD_LOCK_ASSERT(d); 277 278 switch (d->bd_bufmode) { 279 case BPF_BUFMODE_ZBUF: 280 return (bpf_zerocopy_canfreebuf(d)); 281 } 282 return (0); 283} 284 285/* 286 * Allow the buffer model to indicate that the current store buffer is 287 * immutable, regardless of the appearance of space. Return (1) if the 288 * buffer is writable, and (0) if not. 289 */ 290static int 291bpf_canwritebuf(struct bpf_d *d) 292{ 293 294 BPFD_LOCK_ASSERT(d); 295 296 switch (d->bd_bufmode) { 297 case BPF_BUFMODE_ZBUF: 298 return (bpf_zerocopy_canwritebuf(d)); 299 } 300 return (1); 301} 302 303/* 304 * Notify buffer model that an attempt to write to the store buffer has 305 * resulted in a dropped packet, in which case the buffer may be considered 306 * full. 307 */ 308static void 309bpf_buffull(struct bpf_d *d) 310{ 311 312 BPFD_LOCK_ASSERT(d); 313 314 switch (d->bd_bufmode) { 315 case BPF_BUFMODE_ZBUF: 316 bpf_zerocopy_buffull(d); 317 break; 318 } 319} 320 321/* 322 * Notify the buffer model that a buffer has moved into the hold position. 323 */ 324void 325bpf_bufheld(struct bpf_d *d) 326{ 327 328 BPFD_LOCK_ASSERT(d); 329 330 switch (d->bd_bufmode) { 331 case BPF_BUFMODE_ZBUF: 332 bpf_zerocopy_bufheld(d); 333 break; 334 } 335} 336 337static void 338bpf_free(struct bpf_d *d) 339{ 340 341 switch (d->bd_bufmode) { 342 case BPF_BUFMODE_BUFFER: 343 return (bpf_buffer_free(d)); 344 345 case BPF_BUFMODE_ZBUF: 346 return (bpf_zerocopy_free(d)); 347 348 default: 349 panic("bpf_buf_free"); 350 } 351} 352 353static int 354bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio) 355{ 356 357 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) 358 return (EOPNOTSUPP); 359 return (bpf_buffer_uiomove(d, buf, len, uio)); 360} 361 362static int 363bpf_ioctl_sblen(struct bpf_d *d, u_int *i) 364{ 365 366 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) 367 return (EOPNOTSUPP); 368 return (bpf_buffer_ioctl_sblen(d, i)); 369} 370 371static int 372bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i) 373{ 374 375 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 376 return (EOPNOTSUPP); 377 return (bpf_zerocopy_ioctl_getzmax(td, d, i)); 378} 379 380static int 381bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) 382{ 383 384 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 385 return (EOPNOTSUPP); 386 return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz)); 387} 388 389static int 390bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) 391{ 392 393 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 394 return (EOPNOTSUPP); 395 return (bpf_zerocopy_ioctl_setzbuf(td, d, bz)); 396} 397 398/* 399 * General BPF functions. 400 */ 401static int 402bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp, 403 struct sockaddr *sockp, int *hdrlen, struct bpf_insn *wfilter) 404{ 405 const struct ieee80211_bpf_params *p; 406 struct ether_header *eh; 407 struct mbuf *m; 408 int error; 409 int len; 410 int hlen; 411 int slen; 412 413 /* 414 * Build a sockaddr based on the data link layer type. 415 * We do this at this level because the ethernet header 416 * is copied directly into the data field of the sockaddr. 417 * In the case of SLIP, there is no header and the packet 418 * is forwarded as is. 419 * Also, we are careful to leave room at the front of the mbuf 420 * for the link level header. 421 */ 422 switch (linktype) { 423 424 case DLT_SLIP: 425 sockp->sa_family = AF_INET; 426 hlen = 0; 427 break; 428 429 case DLT_EN10MB: 430 sockp->sa_family = AF_UNSPEC; 431 /* XXX Would MAXLINKHDR be better? */ 432 hlen = ETHER_HDR_LEN; 433 break; 434 435 case DLT_FDDI: 436 sockp->sa_family = AF_IMPLINK; 437 hlen = 0; 438 break; 439 440 case DLT_RAW: 441 sockp->sa_family = AF_UNSPEC; 442 hlen = 0; 443 break; 444 445 case DLT_NULL: 446 /* 447 * null interface types require a 4 byte pseudo header which 448 * corresponds to the address family of the packet. 449 */ 450 sockp->sa_family = AF_UNSPEC; 451 hlen = 4; 452 break; 453 454 case DLT_ATM_RFC1483: 455 /* 456 * en atm driver requires 4-byte atm pseudo header. 457 * though it isn't standard, vpi:vci needs to be 458 * specified anyway. 459 */ 460 sockp->sa_family = AF_UNSPEC; 461 hlen = 12; /* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */ 462 break; 463 464 case DLT_PPP: 465 sockp->sa_family = AF_UNSPEC; 466 hlen = 4; /* This should match PPP_HDRLEN */ 467 break; 468 469 case DLT_IEEE802_11: /* IEEE 802.11 wireless */ 470 sockp->sa_family = AF_IEEE80211; 471 hlen = 0; 472 break; 473 474 case DLT_IEEE802_11_RADIO: /* IEEE 802.11 wireless w/ phy params */ 475 sockp->sa_family = AF_IEEE80211; 476 sockp->sa_len = 12; /* XXX != 0 */ 477 hlen = sizeof(struct ieee80211_bpf_params); 478 break; 479 480 default: 481 return (EIO); 482 } 483 484 len = uio->uio_resid; 485 486 if (len - hlen > ifp->if_mtu) 487 return (EMSGSIZE); 488 489 if ((unsigned)len > MJUM16BYTES) 490 return (EIO); 491 492 if (len <= MHLEN) 493 MGETHDR(m, M_WAIT, MT_DATA); 494 else if (len <= MCLBYTES) 495 m = m_getcl(M_WAIT, MT_DATA, M_PKTHDR); 496 else 497 m = m_getjcl(M_WAIT, MT_DATA, M_PKTHDR, 498#if (MJUMPAGESIZE > MCLBYTES) 499 len <= MJUMPAGESIZE ? MJUMPAGESIZE : 500#endif 501 (len <= MJUM9BYTES ? MJUM9BYTES : MJUM16BYTES)); 502 m->m_pkthdr.len = m->m_len = len; 503 m->m_pkthdr.rcvif = NULL; 504 *mp = m; 505 506 if (m->m_len < hlen) { 507 error = EPERM; 508 goto bad; 509 } 510 511 error = uiomove(mtod(m, u_char *), len, uio); 512 if (error) 513 goto bad; 514 515 slen = bpf_filter(wfilter, mtod(m, u_char *), len, len); 516 if (slen == 0) { 517 error = EPERM; 518 goto bad; 519 } 520 521 /* Check for multicast destination */ 522 switch (linktype) { 523 case DLT_EN10MB: 524 eh = mtod(m, struct ether_header *); 525 if (ETHER_IS_MULTICAST(eh->ether_dhost)) { 526 if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost, 527 ETHER_ADDR_LEN) == 0) 528 m->m_flags |= M_BCAST; 529 else 530 m->m_flags |= M_MCAST; 531 } 532 break; 533 } 534 535 /* 536 * Make room for link header, and copy it to sockaddr 537 */ 538 if (hlen != 0) { 539 if (sockp->sa_family == AF_IEEE80211) { 540 /* 541 * Collect true length from the parameter header 542 * NB: sockp is known to be zero'd so if we do a 543 * short copy unspecified parameters will be 544 * zero. 545 * NB: packet may not be aligned after stripping 546 * bpf params 547 * XXX check ibp_vers 548 */ 549 p = mtod(m, const struct ieee80211_bpf_params *); 550 hlen = p->ibp_len; 551 if (hlen > sizeof(sockp->sa_data)) { 552 error = EINVAL; 553 goto bad; 554 } 555 } 556 bcopy(m->m_data, sockp->sa_data, hlen); 557 } 558 *hdrlen = hlen; 559 560 return (0); 561bad: 562 m_freem(m); 563 return (error); 564} 565 566/* 567 * Attach file to the bpf interface, i.e. make d listen on bp. 568 */ 569static void 570bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 571{ 572 /* 573 * Point d at bp, and add d to the interface's list of listeners. 574 * Finally, point the driver's bpf cookie at the interface so 575 * it will divert packets to bpf. 576 */ 577 BPFIF_LOCK(bp); 578 d->bd_bif = bp; 579 LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); 580 581 bpf_bpfd_cnt++; 582 BPFIF_UNLOCK(bp); 583 584 EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1); 585} 586 587/* 588 * Detach a file from its interface. 589 */ 590static void 591bpf_detachd(struct bpf_d *d) 592{ 593 int error; 594 struct bpf_if *bp; 595 struct ifnet *ifp; 596 597 bp = d->bd_bif; 598 BPFIF_LOCK(bp); 599 BPFD_LOCK(d); 600 ifp = d->bd_bif->bif_ifp; 601 602 /* 603 * Remove d from the interface's descriptor list. 604 */ 605 LIST_REMOVE(d, bd_next); 606 607 bpf_bpfd_cnt--; 608 d->bd_bif = NULL; 609 BPFD_UNLOCK(d); 610 BPFIF_UNLOCK(bp); 611 612 EVENTHANDLER_INVOKE(bpf_track, ifp, bp->bif_dlt, 0); 613 614 /* 615 * Check if this descriptor had requested promiscuous mode. 616 * If so, turn it off. 617 */ 618 if (d->bd_promisc) { 619 d->bd_promisc = 0; 620 CURVNET_SET(ifp->if_vnet); 621 error = ifpromisc(ifp, 0); 622 CURVNET_RESTORE(); 623 if (error != 0 && error != ENXIO) { 624 /* 625 * ENXIO can happen if a pccard is unplugged 626 * Something is really wrong if we were able to put 627 * the driver into promiscuous mode, but can't 628 * take it out. 629 */ 630 if_printf(bp->bif_ifp, 631 "bpf_detach: ifpromisc failed (%d)\n", error); 632 } 633 } 634} 635 636/* 637 * Close the descriptor by detaching it from its interface, 638 * deallocating its buffers, and marking it free. 639 */ 640static void 641bpf_dtor(void *data) 642{ 643 struct bpf_d *d = data; 644 645 BPFD_LOCK(d); 646 if (d->bd_state == BPF_WAITING) 647 callout_stop(&d->bd_callout); 648 d->bd_state = BPF_IDLE; 649 BPFD_UNLOCK(d); 650 funsetown(&d->bd_sigio); 651 mtx_lock(&bpf_mtx); 652 if (d->bd_bif) 653 bpf_detachd(d); 654 mtx_unlock(&bpf_mtx); 655#ifdef MAC 656 mac_bpfdesc_destroy(d); 657#endif /* MAC */ 658 seldrain(&d->bd_sel); 659 knlist_destroy(&d->bd_sel.si_note); 660 callout_drain(&d->bd_callout); 661 bpf_freed(d); 662 free(d, M_BPF); 663} 664 665/* 666 * Open ethernet device. Returns ENXIO for illegal minor device number, 667 * EBUSY if file is open by another process. 668 */ 669/* ARGSUSED */ 670static int 671bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td) 672{ 673 struct bpf_d *d; 674 int error; 675 676 d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO); 677 error = devfs_set_cdevpriv(d, bpf_dtor); 678 if (error != 0) { 679 free(d, M_BPF); 680 return (error); 681 } 682 683 /* 684 * For historical reasons, perform a one-time initialization call to 685 * the buffer routines, even though we're not yet committed to a 686 * particular buffer method. 687 */ 688 bpf_buffer_init(d); 689 d->bd_bufmode = BPF_BUFMODE_BUFFER; 690 d->bd_sig = SIGIO; 691 d->bd_direction = BPF_D_INOUT; 692 d->bd_pid = td->td_proc->p_pid; 693#ifdef MAC 694 mac_bpfdesc_init(d); 695 mac_bpfdesc_create(td->td_ucred, d); 696#endif 697 mtx_init(&d->bd_mtx, devtoname(dev), "bpf cdev lock", MTX_DEF); 698 callout_init_mtx(&d->bd_callout, &d->bd_mtx, 0); 699 knlist_init_mtx(&d->bd_sel.si_note, &d->bd_mtx); 700 701 return (0); 702} 703 704/* 705 * bpfread - read next chunk of packets from buffers 706 */ 707static int 708bpfread(struct cdev *dev, struct uio *uio, int ioflag) 709{ 710 struct bpf_d *d; 711 int error; 712 int non_block; 713 int timed_out; 714 715 error = devfs_get_cdevpriv((void **)&d); 716 if (error != 0) 717 return (error); 718 719 /* 720 * Restrict application to use a buffer the same size as 721 * as kernel buffers. 722 */ 723 if (uio->uio_resid != d->bd_bufsize) 724 return (EINVAL); 725 726 non_block = ((ioflag & O_NONBLOCK) != 0); 727 728 BPFD_LOCK(d); 729 d->bd_pid = curthread->td_proc->p_pid; 730 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) { 731 BPFD_UNLOCK(d); 732 return (EOPNOTSUPP); 733 } 734 if (d->bd_state == BPF_WAITING) 735 callout_stop(&d->bd_callout); 736 timed_out = (d->bd_state == BPF_TIMED_OUT); 737 d->bd_state = BPF_IDLE; 738 /* 739 * If the hold buffer is empty, then do a timed sleep, which 740 * ends when the timeout expires or when enough packets 741 * have arrived to fill the store buffer. 742 */ 743 while (d->bd_hbuf == NULL) { 744 if (d->bd_slen != 0) { 745 /* 746 * A packet(s) either arrived since the previous 747 * read or arrived while we were asleep. 748 */ 749 if (d->bd_immediate || non_block || timed_out) { 750 /* 751 * Rotate the buffers and return what's here 752 * if we are in immediate mode, non-blocking 753 * flag is set, or this descriptor timed out. 754 */ 755 ROTATE_BUFFERS(d); 756 break; 757 } 758 } 759 760 /* 761 * No data is available, check to see if the bpf device 762 * is still pointed at a real interface. If not, return 763 * ENXIO so that the userland process knows to rebind 764 * it before using it again. 765 */ 766 if (d->bd_bif == NULL) { 767 BPFD_UNLOCK(d); 768 return (ENXIO); 769 } 770 771 if (non_block) { 772 BPFD_UNLOCK(d); 773 return (EWOULDBLOCK); 774 } 775 error = msleep(d, &d->bd_mtx, PRINET|PCATCH, 776 "bpf", d->bd_rtout); 777 if (error == EINTR || error == ERESTART) { 778 BPFD_UNLOCK(d); 779 return (error); 780 } 781 if (error == EWOULDBLOCK) { 782 /* 783 * On a timeout, return what's in the buffer, 784 * which may be nothing. If there is something 785 * in the store buffer, we can rotate the buffers. 786 */ 787 if (d->bd_hbuf) 788 /* 789 * We filled up the buffer in between 790 * getting the timeout and arriving 791 * here, so we don't need to rotate. 792 */ 793 break; 794 795 if (d->bd_slen == 0) { 796 BPFD_UNLOCK(d); 797 return (0); 798 } 799 ROTATE_BUFFERS(d); 800 break; 801 } 802 } 803 /* 804 * At this point, we know we have something in the hold slot. 805 */ 806 BPFD_UNLOCK(d); 807 808 /* 809 * Move data from hold buffer into user space. 810 * We know the entire buffer is transferred since 811 * we checked above that the read buffer is bpf_bufsize bytes. 812 * 813 * XXXRW: More synchronization needed here: what if a second thread 814 * issues a read on the same fd at the same time? Don't want this 815 * getting invalidated. 816 */ 817 error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio); 818 819 BPFD_LOCK(d); 820 d->bd_fbuf = d->bd_hbuf; 821 d->bd_hbuf = NULL; 822 d->bd_hlen = 0; 823 bpf_buf_reclaimed(d); 824 BPFD_UNLOCK(d); 825 826 return (error); 827} 828 829/* 830 * If there are processes sleeping on this descriptor, wake them up. 831 */ 832static __inline void 833bpf_wakeup(struct bpf_d *d) 834{ 835 836 BPFD_LOCK_ASSERT(d); 837 if (d->bd_state == BPF_WAITING) { 838 callout_stop(&d->bd_callout); 839 d->bd_state = BPF_IDLE; 840 } 841 wakeup(d); 842 if (d->bd_async && d->bd_sig && d->bd_sigio) 843 pgsigio(&d->bd_sigio, d->bd_sig, 0); 844 845 selwakeuppri(&d->bd_sel, PRINET); 846 KNOTE_LOCKED(&d->bd_sel.si_note, 0); 847} 848 849static void 850bpf_timed_out(void *arg) 851{ 852 struct bpf_d *d = (struct bpf_d *)arg; 853 854 BPFD_LOCK_ASSERT(d); 855 856 if (callout_pending(&d->bd_callout) || !callout_active(&d->bd_callout)) 857 return; 858 if (d->bd_state == BPF_WAITING) { 859 d->bd_state = BPF_TIMED_OUT; 860 if (d->bd_slen != 0) 861 bpf_wakeup(d); 862 } 863} 864 865static int 866bpf_ready(struct bpf_d *d) 867{ 868 869 BPFD_LOCK_ASSERT(d); 870 871 if (!bpf_canfreebuf(d) && d->bd_hlen != 0) 872 return (1); 873 if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 874 d->bd_slen != 0) 875 return (1); 876 return (0); 877} 878 879static int 880bpfwrite(struct cdev *dev, struct uio *uio, int ioflag) 881{ 882 struct bpf_d *d; 883 struct ifnet *ifp; 884 struct mbuf *m, *mc; 885 struct sockaddr dst; 886 int error, hlen; 887 888 error = devfs_get_cdevpriv((void **)&d); 889 if (error != 0) 890 return (error); 891 892 d->bd_pid = curthread->td_proc->p_pid; 893 d->bd_wcount++; 894 if (d->bd_bif == NULL) { 895 d->bd_wdcount++; 896 return (ENXIO); 897 } 898 899 ifp = d->bd_bif->bif_ifp; 900 901 if ((ifp->if_flags & IFF_UP) == 0) { 902 d->bd_wdcount++; 903 return (ENETDOWN); 904 } 905 906 if (uio->uio_resid == 0) { 907 d->bd_wdcount++; 908 return (0); 909 } 910 911 bzero(&dst, sizeof(dst)); 912 m = NULL; 913 hlen = 0; 914 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp, 915 &m, &dst, &hlen, d->bd_wfilter); 916 if (error) { 917 d->bd_wdcount++; 918 return (error); 919 } 920 d->bd_wfcount++; 921 if (d->bd_hdrcmplt) 922 dst.sa_family = pseudo_AF_HDRCMPLT; 923 924 if (d->bd_feedback) { 925 mc = m_dup(m, M_DONTWAIT); 926 if (mc != NULL) 927 mc->m_pkthdr.rcvif = ifp; 928 /* Set M_PROMISC for outgoing packets to be discarded. */ 929 if (d->bd_direction == BPF_D_INOUT) 930 m->m_flags |= M_PROMISC; 931 } else 932 mc = NULL; 933 934 m->m_pkthdr.len -= hlen; 935 m->m_len -= hlen; 936 m->m_data += hlen; /* XXX */ 937 938 CURVNET_SET(ifp->if_vnet); 939#ifdef MAC 940 BPFD_LOCK(d); 941 mac_bpfdesc_create_mbuf(d, m); 942 if (mc != NULL) 943 mac_bpfdesc_create_mbuf(d, mc); 944 BPFD_UNLOCK(d); 945#endif 946 947 error = (*ifp->if_output)(ifp, m, &dst, NULL); 948 if (error) 949 d->bd_wdcount++; 950 951 if (mc != NULL) { 952 if (error == 0) 953 (*ifp->if_input)(ifp, mc); 954 else 955 m_freem(mc); 956 } 957 CURVNET_RESTORE(); 958 959 return (error); 960} 961 962/* 963 * Reset a descriptor by flushing its packet buffer and clearing the receive 964 * and drop counts. This is doable for kernel-only buffers, but with 965 * zero-copy buffers, we can't write to (or rotate) buffers that are 966 * currently owned by userspace. It would be nice if we could encapsulate 967 * this logic in the buffer code rather than here. 968 */ 969static void 970reset_d(struct bpf_d *d) 971{ 972 973 mtx_assert(&d->bd_mtx, MA_OWNED); 974 975 if ((d->bd_hbuf != NULL) && 976 (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) { 977 /* Free the hold buffer. */ 978 d->bd_fbuf = d->bd_hbuf; 979 d->bd_hbuf = NULL; 980 d->bd_hlen = 0; 981 bpf_buf_reclaimed(d); 982 } 983 if (bpf_canwritebuf(d)) 984 d->bd_slen = 0; 985 d->bd_rcount = 0; 986 d->bd_dcount = 0; 987 d->bd_fcount = 0; 988 d->bd_wcount = 0; 989 d->bd_wfcount = 0; 990 d->bd_wdcount = 0; 991 d->bd_zcopy = 0; 992} 993 994/* 995 * FIONREAD Check for read packet available. 996 * SIOCGIFADDR Get interface address - convenient hook to driver. 997 * BIOCGBLEN Get buffer len [for read()]. 998 * BIOCSETF Set read filter. 999 * BIOCSETFNR Set read filter without resetting descriptor. 1000 * BIOCSETWF Set write filter. 1001 * BIOCFLUSH Flush read packet buffer. 1002 * BIOCPROMISC Put interface into promiscuous mode. 1003 * BIOCGDLT Get link layer type. 1004 * BIOCGETIF Get interface name. 1005 * BIOCSETIF Set interface. 1006 * BIOCSRTIMEOUT Set read timeout. 1007 * BIOCGRTIMEOUT Get read timeout. 1008 * BIOCGSTATS Get packet stats. 1009 * BIOCIMMEDIATE Set immediate mode. 1010 * BIOCVERSION Get filter language version. 1011 * BIOCGHDRCMPLT Get "header already complete" flag 1012 * BIOCSHDRCMPLT Set "header already complete" flag 1013 * BIOCGDIRECTION Get packet direction flag 1014 * BIOCSDIRECTION Set packet direction flag 1015 * BIOCGTSTAMP Get time stamp format and resolution. 1016 * BIOCSTSTAMP Set time stamp format and resolution. 1017 * BIOCLOCK Set "locked" flag 1018 * BIOCFEEDBACK Set packet feedback mode. 1019 * BIOCSETZBUF Set current zero-copy buffer locations. 1020 * BIOCGETZMAX Get maximum zero-copy buffer size. 1021 * BIOCROTZBUF Force rotation of zero-copy buffer 1022 * BIOCSETBUFMODE Set buffer mode. 1023 * BIOCGETBUFMODE Get current buffer mode. 1024 */ 1025/* ARGSUSED */ 1026static int 1027bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, 1028 struct thread *td) 1029{ 1030 struct bpf_d *d; 1031 int error; 1032 1033 error = devfs_get_cdevpriv((void **)&d); 1034 if (error != 0) 1035 return (error); 1036 1037 /* 1038 * Refresh PID associated with this descriptor. 1039 */ 1040 BPFD_LOCK(d); 1041 d->bd_pid = td->td_proc->p_pid; 1042 if (d->bd_state == BPF_WAITING) 1043 callout_stop(&d->bd_callout); 1044 d->bd_state = BPF_IDLE; 1045 BPFD_UNLOCK(d); 1046 1047 if (d->bd_locked == 1) { 1048 switch (cmd) { 1049 case BIOCGBLEN: 1050 case BIOCFLUSH: 1051 case BIOCGDLT: 1052 case BIOCGDLTLIST: 1053#ifdef COMPAT_FREEBSD32 1054 case BIOCGDLTLIST32: 1055#endif 1056 case BIOCGETIF: 1057 case BIOCGRTIMEOUT: 1058#ifdef COMPAT_FREEBSD32 1059 case BIOCGRTIMEOUT32: 1060#endif 1061 case BIOCGSTATS: 1062 case BIOCVERSION: 1063 case BIOCGRSIG: 1064 case BIOCGHDRCMPLT: 1065 case BIOCSTSTAMP: 1066 case BIOCFEEDBACK: 1067 case FIONREAD: 1068 case BIOCLOCK: 1069 case BIOCSRTIMEOUT: 1070#ifdef COMPAT_FREEBSD32 1071 case BIOCSRTIMEOUT32: 1072#endif 1073 case BIOCIMMEDIATE: 1074 case TIOCGPGRP: 1075 case BIOCROTZBUF: 1076 break; 1077 default: 1078 return (EPERM); 1079 } 1080 } 1081#ifdef COMPAT_FREEBSD32 1082 /* 1083 * If we see a 32-bit compat ioctl, mark the stream as 32-bit so 1084 * that it will get 32-bit packet headers. 1085 */ 1086 switch (cmd) { 1087 case BIOCSETF32: 1088 case BIOCSETFNR32: 1089 case BIOCSETWF32: 1090 case BIOCGDLTLIST32: 1091 case BIOCGRTIMEOUT32: 1092 case BIOCSRTIMEOUT32: 1093 d->bd_compat32 = 1; 1094 } 1095#endif 1096 1097 CURVNET_SET(TD_TO_VNET(td)); 1098 switch (cmd) { 1099 1100 default: 1101 error = EINVAL; 1102 break; 1103 1104 /* 1105 * Check for read packet available. 1106 */ 1107 case FIONREAD: 1108 { 1109 int n; 1110 1111 BPFD_LOCK(d); 1112 n = d->bd_slen; 1113 if (d->bd_hbuf) 1114 n += d->bd_hlen; 1115 BPFD_UNLOCK(d); 1116 1117 *(int *)addr = n; 1118 break; 1119 } 1120 1121 case SIOCGIFADDR: 1122 { 1123 struct ifnet *ifp; 1124 1125 if (d->bd_bif == NULL) 1126 error = EINVAL; 1127 else { 1128 ifp = d->bd_bif->bif_ifp; 1129 error = (*ifp->if_ioctl)(ifp, cmd, addr); 1130 } 1131 break; 1132 } 1133 1134 /* 1135 * Get buffer len [for read()]. 1136 */ 1137 case BIOCGBLEN: 1138 *(u_int *)addr = d->bd_bufsize; 1139 break; 1140 1141 /* 1142 * Set buffer length. 1143 */ 1144 case BIOCSBLEN: 1145 error = bpf_ioctl_sblen(d, (u_int *)addr); 1146 break; 1147 1148 /* 1149 * Set link layer read filter. 1150 */ 1151 case BIOCSETF: 1152 case BIOCSETFNR: 1153 case BIOCSETWF: 1154#ifdef COMPAT_FREEBSD32 1155 case BIOCSETF32: 1156 case BIOCSETFNR32: 1157 case BIOCSETWF32: 1158#endif 1159 error = bpf_setf(d, (struct bpf_program *)addr, cmd); 1160 break; 1161 1162 /* 1163 * Flush read packet buffer. 1164 */ 1165 case BIOCFLUSH: 1166 BPFD_LOCK(d); 1167 reset_d(d); 1168 BPFD_UNLOCK(d); 1169 break; 1170 1171 /* 1172 * Put interface into promiscuous mode. 1173 */ 1174 case BIOCPROMISC: 1175 if (d->bd_bif == NULL) { 1176 /* 1177 * No interface attached yet. 1178 */ 1179 error = EINVAL; 1180 break; 1181 } 1182 if (d->bd_promisc == 0) { 1183 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1184 if (error == 0) 1185 d->bd_promisc = 1; 1186 } 1187 break; 1188 1189 /* 1190 * Get current data link type. 1191 */ 1192 case BIOCGDLT: 1193 if (d->bd_bif == NULL) 1194 error = EINVAL; 1195 else 1196 *(u_int *)addr = d->bd_bif->bif_dlt; 1197 break; 1198 1199 /* 1200 * Get a list of supported data link types. 1201 */ 1202#ifdef COMPAT_FREEBSD32 1203 case BIOCGDLTLIST32: 1204 { 1205 struct bpf_dltlist32 *list32; 1206 struct bpf_dltlist dltlist; 1207 1208 list32 = (struct bpf_dltlist32 *)addr; 1209 dltlist.bfl_len = list32->bfl_len; 1210 dltlist.bfl_list = PTRIN(list32->bfl_list); 1211 if (d->bd_bif == NULL) 1212 error = EINVAL; 1213 else { 1214 error = bpf_getdltlist(d, &dltlist); 1215 if (error == 0) 1216 list32->bfl_len = dltlist.bfl_len; 1217 } 1218 break; 1219 } 1220#endif 1221 1222 case BIOCGDLTLIST: 1223 if (d->bd_bif == NULL) 1224 error = EINVAL; 1225 else 1226 error = bpf_getdltlist(d, (struct bpf_dltlist *)addr); 1227 break; 1228 1229 /* 1230 * Set data link type. 1231 */ 1232 case BIOCSDLT: 1233 if (d->bd_bif == NULL) 1234 error = EINVAL; 1235 else 1236 error = bpf_setdlt(d, *(u_int *)addr); 1237 break; 1238 1239 /* 1240 * Get interface name. 1241 */ 1242 case BIOCGETIF: 1243 if (d->bd_bif == NULL) 1244 error = EINVAL; 1245 else { 1246 struct ifnet *const ifp = d->bd_bif->bif_ifp; 1247 struct ifreq *const ifr = (struct ifreq *)addr; 1248 1249 strlcpy(ifr->ifr_name, ifp->if_xname, 1250 sizeof(ifr->ifr_name)); 1251 } 1252 break; 1253 1254 /* 1255 * Set interface. 1256 */ 1257 case BIOCSETIF: 1258 error = bpf_setif(d, (struct ifreq *)addr); 1259 break; 1260 1261 /* 1262 * Set read timeout. 1263 */ 1264 case BIOCSRTIMEOUT: 1265#ifdef COMPAT_FREEBSD32 1266 case BIOCSRTIMEOUT32: 1267#endif 1268 { 1269 struct timeval *tv = (struct timeval *)addr; 1270#ifdef COMPAT_FREEBSD32 1271 struct timeval32 *tv32; 1272 struct timeval tv64; 1273 1274 if (cmd == BIOCSRTIMEOUT32) { 1275 tv32 = (struct timeval32 *)addr; 1276 tv = &tv64; 1277 tv->tv_sec = tv32->tv_sec; 1278 tv->tv_usec = tv32->tv_usec; 1279 } else 1280#endif 1281 tv = (struct timeval *)addr; 1282 1283 /* 1284 * Subtract 1 tick from tvtohz() since this isn't 1285 * a one-shot timer. 1286 */ 1287 if ((error = itimerfix(tv)) == 0) 1288 d->bd_rtout = tvtohz(tv) - 1; 1289 break; 1290 } 1291 1292 /* 1293 * Get read timeout. 1294 */ 1295 case BIOCGRTIMEOUT: 1296#ifdef COMPAT_FREEBSD32 1297 case BIOCGRTIMEOUT32: 1298#endif 1299 { 1300 struct timeval *tv; 1301#ifdef COMPAT_FREEBSD32 1302 struct timeval32 *tv32; 1303 struct timeval tv64; 1304 1305 if (cmd == BIOCGRTIMEOUT32) 1306 tv = &tv64; 1307 else 1308#endif 1309 tv = (struct timeval *)addr; 1310 1311 tv->tv_sec = d->bd_rtout / hz; 1312 tv->tv_usec = (d->bd_rtout % hz) * tick; 1313#ifdef COMPAT_FREEBSD32 1314 if (cmd == BIOCGRTIMEOUT32) { 1315 tv32 = (struct timeval32 *)addr; 1316 tv32->tv_sec = tv->tv_sec; 1317 tv32->tv_usec = tv->tv_usec; 1318 } 1319#endif 1320 1321 break; 1322 } 1323 1324 /* 1325 * Get packet stats. 1326 */ 1327 case BIOCGSTATS: 1328 { 1329 struct bpf_stat *bs = (struct bpf_stat *)addr; 1330 1331 /* XXXCSJP overflow */ 1332 bs->bs_recv = d->bd_rcount; 1333 bs->bs_drop = d->bd_dcount; 1334 break; 1335 } 1336 1337 /* 1338 * Set immediate mode. 1339 */ 1340 case BIOCIMMEDIATE: 1341 d->bd_immediate = *(u_int *)addr; 1342 break; 1343 1344 case BIOCVERSION: 1345 { 1346 struct bpf_version *bv = (struct bpf_version *)addr; 1347 1348 bv->bv_major = BPF_MAJOR_VERSION; 1349 bv->bv_minor = BPF_MINOR_VERSION; 1350 break; 1351 } 1352 1353 /* 1354 * Get "header already complete" flag 1355 */ 1356 case BIOCGHDRCMPLT: 1357 *(u_int *)addr = d->bd_hdrcmplt; 1358 break; 1359 1360 /* 1361 * Set "header already complete" flag 1362 */ 1363 case BIOCSHDRCMPLT: 1364 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1365 break; 1366 1367 /* 1368 * Get packet direction flag 1369 */ 1370 case BIOCGDIRECTION: 1371 *(u_int *)addr = d->bd_direction; 1372 break; 1373 1374 /* 1375 * Set packet direction flag 1376 */ 1377 case BIOCSDIRECTION: 1378 { 1379 u_int direction; 1380 1381 direction = *(u_int *)addr; 1382 switch (direction) { 1383 case BPF_D_IN: 1384 case BPF_D_INOUT: 1385 case BPF_D_OUT: 1386 d->bd_direction = direction; 1387 break; 1388 default: 1389 error = EINVAL; 1390 } 1391 } 1392 break; 1393 1394 /* 1395 * Get packet timestamp format and resolution. 1396 */ 1397 case BIOCGTSTAMP: 1398 *(u_int *)addr = d->bd_tstamp; 1399 break; 1400 1401 /* 1402 * Set packet timestamp format and resolution. 1403 */ 1404 case BIOCSTSTAMP: 1405 { 1406 u_int func; 1407 1408 func = *(u_int *)addr; 1409 if (BPF_T_VALID(func)) 1410 d->bd_tstamp = func; 1411 else 1412 error = EINVAL; 1413 } 1414 break; 1415 1416 case BIOCFEEDBACK: 1417 d->bd_feedback = *(u_int *)addr; 1418 break; 1419 1420 case BIOCLOCK: 1421 d->bd_locked = 1; 1422 break; 1423 1424 case FIONBIO: /* Non-blocking I/O */ 1425 break; 1426 1427 case FIOASYNC: /* Send signal on receive packets */ 1428 d->bd_async = *(int *)addr; 1429 break; 1430 1431 case FIOSETOWN: 1432 error = fsetown(*(int *)addr, &d->bd_sigio); 1433 break; 1434 1435 case FIOGETOWN: 1436 *(int *)addr = fgetown(&d->bd_sigio); 1437 break; 1438 1439 /* This is deprecated, FIOSETOWN should be used instead. */ 1440 case TIOCSPGRP: 1441 error = fsetown(-(*(int *)addr), &d->bd_sigio); 1442 break; 1443 1444 /* This is deprecated, FIOGETOWN should be used instead. */ 1445 case TIOCGPGRP: 1446 *(int *)addr = -fgetown(&d->bd_sigio); 1447 break; 1448 1449 case BIOCSRSIG: /* Set receive signal */ 1450 { 1451 u_int sig; 1452 1453 sig = *(u_int *)addr; 1454 1455 if (sig >= NSIG) 1456 error = EINVAL; 1457 else 1458 d->bd_sig = sig; 1459 break; 1460 } 1461 case BIOCGRSIG: 1462 *(u_int *)addr = d->bd_sig; 1463 break; 1464 1465 case BIOCGETBUFMODE: 1466 *(u_int *)addr = d->bd_bufmode; 1467 break; 1468 1469 case BIOCSETBUFMODE: 1470 /* 1471 * Allow the buffering mode to be changed as long as we 1472 * haven't yet committed to a particular mode. Our 1473 * definition of commitment, for now, is whether or not a 1474 * buffer has been allocated or an interface attached, since 1475 * that's the point where things get tricky. 1476 */ 1477 switch (*(u_int *)addr) { 1478 case BPF_BUFMODE_BUFFER: 1479 break; 1480 1481 case BPF_BUFMODE_ZBUF: 1482 if (bpf_zerocopy_enable) 1483 break; 1484 /* FALLSTHROUGH */ 1485 1486 default: 1487 CURVNET_RESTORE(); 1488 return (EINVAL); 1489 } 1490 1491 BPFD_LOCK(d); 1492 if (d->bd_sbuf != NULL || d->bd_hbuf != NULL || 1493 d->bd_fbuf != NULL || d->bd_bif != NULL) { 1494 BPFD_UNLOCK(d); 1495 CURVNET_RESTORE(); 1496 return (EBUSY); 1497 } 1498 d->bd_bufmode = *(u_int *)addr; 1499 BPFD_UNLOCK(d); 1500 break; 1501 1502 case BIOCGETZMAX: 1503 error = bpf_ioctl_getzmax(td, d, (size_t *)addr); 1504 break; 1505 1506 case BIOCSETZBUF: 1507 error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr); 1508 break; 1509 1510 case BIOCROTZBUF: 1511 error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr); 1512 break; 1513 } 1514 CURVNET_RESTORE(); 1515 return (error); 1516} 1517 1518/* 1519 * Set d's packet filter program to fp. If this file already has a filter, 1520 * free it and replace it. Returns EINVAL for bogus requests. 1521 */ 1522static int 1523bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) 1524{ 1525 struct bpf_insn *fcode, *old; 1526 u_int wfilter, flen, size; 1527#ifdef BPF_JITTER 1528 bpf_jit_filter *ofunc; 1529#endif 1530#ifdef COMPAT_FREEBSD32 1531 struct bpf_program32 *fp32; 1532 struct bpf_program fp_swab; 1533 1534 if (cmd == BIOCSETWF32 || cmd == BIOCSETF32 || cmd == BIOCSETFNR32) { 1535 fp32 = (struct bpf_program32 *)fp; 1536 fp_swab.bf_len = fp32->bf_len; 1537 fp_swab.bf_insns = (struct bpf_insn *)(uintptr_t)fp32->bf_insns; 1538 fp = &fp_swab; 1539 if (cmd == BIOCSETWF32) 1540 cmd = BIOCSETWF; 1541 } 1542#endif 1543 if (cmd == BIOCSETWF) { 1544 old = d->bd_wfilter; 1545 wfilter = 1; 1546#ifdef BPF_JITTER 1547 ofunc = NULL; 1548#endif 1549 } else { 1550 wfilter = 0; 1551 old = d->bd_rfilter; 1552#ifdef BPF_JITTER 1553 ofunc = d->bd_bfilter; 1554#endif 1555 } 1556 if (fp->bf_insns == NULL) { 1557 if (fp->bf_len != 0) 1558 return (EINVAL); 1559 BPFD_LOCK(d); 1560 if (wfilter) 1561 d->bd_wfilter = NULL; 1562 else { 1563 d->bd_rfilter = NULL; 1564#ifdef BPF_JITTER 1565 d->bd_bfilter = NULL; 1566#endif 1567 if (cmd == BIOCSETF) 1568 reset_d(d); 1569 } 1570 BPFD_UNLOCK(d); 1571 if (old != NULL) 1572 free((caddr_t)old, M_BPF); 1573#ifdef BPF_JITTER 1574 if (ofunc != NULL) 1575 bpf_destroy_jit_filter(ofunc); 1576#endif 1577 return (0); 1578 } 1579 flen = fp->bf_len; 1580 if (flen > bpf_maxinsns) 1581 return (EINVAL); 1582 1583 size = flen * sizeof(*fp->bf_insns); 1584 fcode = (struct bpf_insn *)malloc(size, M_BPF, M_WAITOK); 1585 if (copyin((caddr_t)fp->bf_insns, (caddr_t)fcode, size) == 0 && 1586 bpf_validate(fcode, (int)flen)) { 1587 BPFD_LOCK(d); 1588 if (wfilter) 1589 d->bd_wfilter = fcode; 1590 else { 1591 d->bd_rfilter = fcode; 1592#ifdef BPF_JITTER 1593 d->bd_bfilter = bpf_jitter(fcode, flen); 1594#endif 1595 if (cmd == BIOCSETF) 1596 reset_d(d); 1597 } 1598 BPFD_UNLOCK(d); 1599 if (old != NULL) 1600 free((caddr_t)old, M_BPF); 1601#ifdef BPF_JITTER 1602 if (ofunc != NULL) 1603 bpf_destroy_jit_filter(ofunc); 1604#endif 1605 1606 return (0); 1607 } 1608 free((caddr_t)fcode, M_BPF); 1609 return (EINVAL); 1610} 1611 1612/* 1613 * Detach a file from its current interface (if attached at all) and attach 1614 * to the interface indicated by the name stored in ifr. 1615 * Return an errno or 0. 1616 */ 1617static int 1618bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1619{ 1620 struct bpf_if *bp; 1621 struct ifnet *theywant; 1622 1623 theywant = ifunit(ifr->ifr_name); 1624 if (theywant == NULL || theywant->if_bpf == NULL) 1625 return (ENXIO); 1626 1627 bp = theywant->if_bpf; 1628 1629 /* 1630 * Behavior here depends on the buffering model. If we're using 1631 * kernel memory buffers, then we can allocate them here. If we're 1632 * using zero-copy, then the user process must have registered 1633 * buffers by the time we get here. If not, return an error. 1634 * 1635 * XXXRW: There are locking issues here with multi-threaded use: what 1636 * if two threads try to set the interface at once? 1637 */ 1638 switch (d->bd_bufmode) { 1639 case BPF_BUFMODE_BUFFER: 1640 if (d->bd_sbuf == NULL) 1641 bpf_buffer_alloc(d); 1642 KASSERT(d->bd_sbuf != NULL, ("bpf_setif: bd_sbuf NULL")); 1643 break; 1644 1645 case BPF_BUFMODE_ZBUF: 1646 if (d->bd_sbuf == NULL) 1647 return (EINVAL); 1648 break; 1649 1650 default: 1651 panic("bpf_setif: bufmode %d", d->bd_bufmode); 1652 } 1653 if (bp != d->bd_bif) { 1654 if (d->bd_bif) 1655 /* 1656 * Detach if attached to something else. 1657 */ 1658 bpf_detachd(d); 1659 1660 bpf_attachd(d, bp); 1661 } 1662 BPFD_LOCK(d); 1663 reset_d(d); 1664 BPFD_UNLOCK(d); 1665 return (0); 1666} 1667 1668/* 1669 * Support for select() and poll() system calls 1670 * 1671 * Return true iff the specific operation will not block indefinitely. 1672 * Otherwise, return false but make a note that a selwakeup() must be done. 1673 */ 1674static int 1675bpfpoll(struct cdev *dev, int events, struct thread *td) 1676{ 1677 struct bpf_d *d; 1678 int revents; 1679 1680 if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL) 1681 return (events & 1682 (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM)); 1683 1684 /* 1685 * Refresh PID associated with this descriptor. 1686 */ 1687 revents = events & (POLLOUT | POLLWRNORM); 1688 BPFD_LOCK(d); 1689 d->bd_pid = td->td_proc->p_pid; 1690 if (events & (POLLIN | POLLRDNORM)) { 1691 if (bpf_ready(d)) 1692 revents |= events & (POLLIN | POLLRDNORM); 1693 else { 1694 selrecord(td, &d->bd_sel); 1695 /* Start the read timeout if necessary. */ 1696 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1697 callout_reset(&d->bd_callout, d->bd_rtout, 1698 bpf_timed_out, d); 1699 d->bd_state = BPF_WAITING; 1700 } 1701 } 1702 } 1703 BPFD_UNLOCK(d); 1704 return (revents); 1705} 1706 1707/* 1708 * Support for kevent() system call. Register EVFILT_READ filters and 1709 * reject all others. 1710 */ 1711int 1712bpfkqfilter(struct cdev *dev, struct knote *kn) 1713{ 1714 struct bpf_d *d; 1715 1716 if (devfs_get_cdevpriv((void **)&d) != 0 || 1717 kn->kn_filter != EVFILT_READ) 1718 return (1); 1719 1720 /* 1721 * Refresh PID associated with this descriptor. 1722 */ 1723 BPFD_LOCK(d); 1724 d->bd_pid = curthread->td_proc->p_pid; 1725 kn->kn_fop = &bpfread_filtops; 1726 kn->kn_hook = d; 1727 knlist_add(&d->bd_sel.si_note, kn, 1); 1728 BPFD_UNLOCK(d); 1729 1730 return (0); 1731} 1732 1733static void 1734filt_bpfdetach(struct knote *kn) 1735{ 1736 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; 1737 1738 knlist_remove(&d->bd_sel.si_note, kn, 0); 1739} 1740 1741static int 1742filt_bpfread(struct knote *kn, long hint) 1743{ 1744 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; 1745 int ready; 1746 1747 BPFD_LOCK_ASSERT(d); 1748 ready = bpf_ready(d); 1749 if (ready) { 1750 kn->kn_data = d->bd_slen; 1751 if (d->bd_hbuf) 1752 kn->kn_data += d->bd_hlen; 1753 } else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1754 callout_reset(&d->bd_callout, d->bd_rtout, 1755 bpf_timed_out, d); 1756 d->bd_state = BPF_WAITING; 1757 } 1758 1759 return (ready); 1760} 1761 1762#define BPF_TSTAMP_NONE 0 1763#define BPF_TSTAMP_FAST 1 1764#define BPF_TSTAMP_NORMAL 2 1765#define BPF_TSTAMP_EXTERN 3 1766 1767static int 1768bpf_ts_quality(int tstype) 1769{ 1770 1771 if (tstype == BPF_T_NONE) 1772 return (BPF_TSTAMP_NONE); 1773 if ((tstype & BPF_T_FAST) != 0) 1774 return (BPF_TSTAMP_FAST); 1775 1776 return (BPF_TSTAMP_NORMAL); 1777} 1778 1779static int 1780bpf_gettime(struct bintime *bt, int tstype, struct mbuf *m) 1781{ 1782 struct m_tag *tag; 1783 int quality; 1784 1785 quality = bpf_ts_quality(tstype); 1786 if (quality == BPF_TSTAMP_NONE) 1787 return (quality); 1788 1789 if (m != NULL) { 1790 tag = m_tag_locate(m, MTAG_BPF, MTAG_BPF_TIMESTAMP, NULL); 1791 if (tag != NULL) { 1792 *bt = *(struct bintime *)(tag + 1); 1793 return (BPF_TSTAMP_EXTERN); 1794 } 1795 } 1796 if (quality == BPF_TSTAMP_NORMAL) 1797 binuptime(bt); 1798 else 1799 getbinuptime(bt); 1800 1801 return (quality); 1802} 1803 1804/* 1805 * Incoming linkage from device drivers. Process the packet pkt, of length 1806 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1807 * by each process' filter, and if accepted, stashed into the corresponding 1808 * buffer. 1809 */ 1810void 1811bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1812{ 1813 struct bintime bt; 1814 struct bpf_d *d; 1815#ifdef BPF_JITTER 1816 bpf_jit_filter *bf; 1817#endif 1818 u_int slen; 1819 int gottime; 1820 1821 gottime = BPF_TSTAMP_NONE; 1822 BPFIF_LOCK(bp); 1823 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 1824 BPFD_LOCK(d); 1825 ++d->bd_rcount; 1826 /* 1827 * NB: We dont call BPF_CHECK_DIRECTION() here since there is no 1828 * way for the caller to indiciate to us whether this packet 1829 * is inbound or outbound. In the bpf_mtap() routines, we use 1830 * the interface pointers on the mbuf to figure it out. 1831 */ 1832#ifdef BPF_JITTER 1833 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; 1834 if (bf != NULL) 1835 slen = (*(bf->func))(pkt, pktlen, pktlen); 1836 else 1837#endif 1838 slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen); 1839 if (slen != 0) { 1840 d->bd_fcount++; 1841 if (gottime < bpf_ts_quality(d->bd_tstamp)) 1842 gottime = bpf_gettime(&bt, d->bd_tstamp, NULL); 1843#ifdef MAC 1844 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 1845#endif 1846 catchpacket(d, pkt, pktlen, slen, 1847 bpf_append_bytes, &bt); 1848 } 1849 BPFD_UNLOCK(d); 1850 } 1851 BPFIF_UNLOCK(bp); 1852} 1853 1854#define BPF_CHECK_DIRECTION(d, r, i) \ 1855 (((d)->bd_direction == BPF_D_IN && (r) != (i)) || \ 1856 ((d)->bd_direction == BPF_D_OUT && (r) == (i))) 1857 1858/* 1859 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1860 */ 1861void 1862bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1863{ 1864 struct bintime bt; 1865 struct bpf_d *d; 1866#ifdef BPF_JITTER 1867 bpf_jit_filter *bf; 1868#endif 1869 u_int pktlen, slen; 1870 int gottime; 1871 1872 /* Skip outgoing duplicate packets. */ 1873 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1874 m->m_flags &= ~M_PROMISC; 1875 return; 1876 } 1877 1878 pktlen = m_length(m, NULL); 1879 1880 gottime = BPF_TSTAMP_NONE; 1881 BPFIF_LOCK(bp); 1882 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 1883 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) 1884 continue; 1885 BPFD_LOCK(d); 1886 ++d->bd_rcount; 1887#ifdef BPF_JITTER 1888 bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL; 1889 /* XXX We cannot handle multiple mbufs. */ 1890 if (bf != NULL && m->m_next == NULL) 1891 slen = (*(bf->func))(mtod(m, u_char *), pktlen, pktlen); 1892 else 1893#endif 1894 slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0); 1895 if (slen != 0) { 1896 d->bd_fcount++; 1897 if (gottime < bpf_ts_quality(d->bd_tstamp)) 1898 gottime = bpf_gettime(&bt, d->bd_tstamp, m); 1899#ifdef MAC 1900 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 1901#endif 1902 catchpacket(d, (u_char *)m, pktlen, slen, 1903 bpf_append_mbuf, &bt); 1904 } 1905 BPFD_UNLOCK(d); 1906 } 1907 BPFIF_UNLOCK(bp); 1908} 1909 1910/* 1911 * Incoming linkage from device drivers, when packet is in 1912 * an mbuf chain and to be prepended by a contiguous header. 1913 */ 1914void 1915bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1916{ 1917 struct bintime bt; 1918 struct mbuf mb; 1919 struct bpf_d *d; 1920 u_int pktlen, slen; 1921 int gottime; 1922 1923 /* Skip outgoing duplicate packets. */ 1924 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1925 m->m_flags &= ~M_PROMISC; 1926 return; 1927 } 1928 1929 pktlen = m_length(m, NULL); 1930 /* 1931 * Craft on-stack mbuf suitable for passing to bpf_filter. 1932 * Note that we cut corners here; we only setup what's 1933 * absolutely needed--this mbuf should never go anywhere else. 1934 */ 1935 mb.m_next = m; 1936 mb.m_data = data; 1937 mb.m_len = dlen; 1938 pktlen += dlen; 1939 1940 gottime = BPF_TSTAMP_NONE; 1941 BPFIF_LOCK(bp); 1942 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 1943 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) 1944 continue; 1945 BPFD_LOCK(d); 1946 ++d->bd_rcount; 1947 slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); 1948 if (slen != 0) { 1949 d->bd_fcount++; 1950 if (gottime < bpf_ts_quality(d->bd_tstamp)) 1951 gottime = bpf_gettime(&bt, d->bd_tstamp, m); 1952#ifdef MAC 1953 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 1954#endif 1955 catchpacket(d, (u_char *)&mb, pktlen, slen, 1956 bpf_append_mbuf, &bt); 1957 } 1958 BPFD_UNLOCK(d); 1959 } 1960 BPFIF_UNLOCK(bp); 1961} 1962 1963#undef BPF_CHECK_DIRECTION 1964 1965#undef BPF_TSTAMP_NONE 1966#undef BPF_TSTAMP_FAST 1967#undef BPF_TSTAMP_NORMAL 1968#undef BPF_TSTAMP_EXTERN 1969 1970static int 1971bpf_hdrlen(struct bpf_d *d) 1972{ 1973 int hdrlen; 1974 1975 hdrlen = d->bd_bif->bif_hdrlen; 1976#ifndef BURN_BRIDGES 1977 if (d->bd_tstamp == BPF_T_NONE || 1978 BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME) 1979#ifdef COMPAT_FREEBSD32 1980 if (d->bd_compat32) 1981 hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr32); 1982 else 1983#endif 1984 hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr); 1985 else 1986#endif 1987 hdrlen += SIZEOF_BPF_HDR(struct bpf_xhdr); 1988#ifdef COMPAT_FREEBSD32 1989 if (d->bd_compat32) 1990 hdrlen = BPF_WORDALIGN32(hdrlen); 1991 else 1992#endif 1993 hdrlen = BPF_WORDALIGN(hdrlen); 1994 1995 return (hdrlen - d->bd_bif->bif_hdrlen); 1996} 1997 1998static void 1999bpf_bintime2ts(struct bintime *bt, struct bpf_ts *ts, int tstype) 2000{ 2001 struct bintime bt2; 2002 struct timeval tsm; 2003 struct timespec tsn; 2004 2005 if ((tstype & BPF_T_MONOTONIC) == 0) { 2006 bt2 = *bt; 2007 bintime_add(&bt2, &boottimebin); 2008 bt = &bt2; 2009 } 2010 switch (BPF_T_FORMAT(tstype)) { 2011 case BPF_T_MICROTIME: 2012 bintime2timeval(bt, &tsm); 2013 ts->bt_sec = tsm.tv_sec; 2014 ts->bt_frac = tsm.tv_usec; 2015 break; 2016 case BPF_T_NANOTIME: 2017 bintime2timespec(bt, &tsn); 2018 ts->bt_sec = tsn.tv_sec; 2019 ts->bt_frac = tsn.tv_nsec; 2020 break; 2021 case BPF_T_BINTIME: 2022 ts->bt_sec = bt->sec; 2023 ts->bt_frac = bt->frac; 2024 break; 2025 } 2026} 2027 2028/* 2029 * Move the packet data from interface memory (pkt) into the 2030 * store buffer. "cpfn" is the routine called to do the actual data 2031 * transfer. bcopy is passed in to copy contiguous chunks, while 2032 * bpf_append_mbuf is passed in to copy mbuf chains. In the latter case, 2033 * pkt is really an mbuf. 2034 */ 2035static void 2036catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 2037 void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int), 2038 struct bintime *bt) 2039{ 2040 struct bpf_xhdr hdr; 2041#ifndef BURN_BRIDGES 2042 struct bpf_hdr hdr_old; 2043#ifdef COMPAT_FREEBSD32 2044 struct bpf_hdr32 hdr32_old; 2045#endif 2046#endif 2047 int caplen, curlen, hdrlen, totlen; 2048 int do_wakeup = 0; 2049 int do_timestamp; 2050 int tstype; 2051 2052 BPFD_LOCK_ASSERT(d); 2053 2054 /* 2055 * Detect whether user space has released a buffer back to us, and if 2056 * so, move it from being a hold buffer to a free buffer. This may 2057 * not be the best place to do it (for example, we might only want to 2058 * run this check if we need the space), but for now it's a reliable 2059 * spot to do it. 2060 */ 2061 if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) { 2062 d->bd_fbuf = d->bd_hbuf; 2063 d->bd_hbuf = NULL; 2064 d->bd_hlen = 0; 2065 bpf_buf_reclaimed(d); 2066 } 2067 2068 /* 2069 * Figure out how many bytes to move. If the packet is 2070 * greater or equal to the snapshot length, transfer that 2071 * much. Otherwise, transfer the whole packet (unless 2072 * we hit the buffer size limit). 2073 */ 2074 hdrlen = bpf_hdrlen(d); 2075 totlen = hdrlen + min(snaplen, pktlen); 2076 if (totlen > d->bd_bufsize) 2077 totlen = d->bd_bufsize; 2078 2079 /* 2080 * Round up the end of the previous packet to the next longword. 2081 * 2082 * Drop the packet if there's no room and no hope of room 2083 * If the packet would overflow the storage buffer or the storage 2084 * buffer is considered immutable by the buffer model, try to rotate 2085 * the buffer and wakeup pending processes. 2086 */ 2087#ifdef COMPAT_FREEBSD32 2088 if (d->bd_compat32) 2089 curlen = BPF_WORDALIGN32(d->bd_slen); 2090 else 2091#endif 2092 curlen = BPF_WORDALIGN(d->bd_slen); 2093 if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) { 2094 if (d->bd_fbuf == NULL) { 2095 /* 2096 * There's no room in the store buffer, and no 2097 * prospect of room, so drop the packet. Notify the 2098 * buffer model. 2099 */ 2100 bpf_buffull(d); 2101 ++d->bd_dcount; 2102 return; 2103 } 2104 ROTATE_BUFFERS(d); 2105 do_wakeup = 1; 2106 curlen = 0; 2107 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) 2108 /* 2109 * Immediate mode is set, or the read timeout has already 2110 * expired during a select call. A packet arrived, so the 2111 * reader should be woken up. 2112 */ 2113 do_wakeup = 1; 2114 caplen = totlen - hdrlen; 2115 tstype = d->bd_tstamp; 2116 do_timestamp = tstype != BPF_T_NONE; 2117#ifndef BURN_BRIDGES 2118 if (tstype == BPF_T_NONE || BPF_T_FORMAT(tstype) == BPF_T_MICROTIME) { 2119 struct bpf_ts ts; 2120 if (do_timestamp) 2121 bpf_bintime2ts(bt, &ts, tstype); 2122#ifdef COMPAT_FREEBSD32 2123 if (d->bd_compat32) { 2124 bzero(&hdr32_old, sizeof(hdr32_old)); 2125 if (do_timestamp) { 2126 hdr32_old.bh_tstamp.tv_sec = ts.bt_sec; 2127 hdr32_old.bh_tstamp.tv_usec = ts.bt_frac; 2128 } 2129 hdr32_old.bh_datalen = pktlen; 2130 hdr32_old.bh_hdrlen = hdrlen; 2131 hdr32_old.bh_caplen = caplen; 2132 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old, 2133 sizeof(hdr32_old)); 2134 goto copy; 2135 } 2136#endif 2137 bzero(&hdr_old, sizeof(hdr_old)); 2138 if (do_timestamp) { 2139 hdr_old.bh_tstamp.tv_sec = ts.bt_sec; 2140 hdr_old.bh_tstamp.tv_usec = ts.bt_frac; 2141 } 2142 hdr_old.bh_datalen = pktlen; 2143 hdr_old.bh_hdrlen = hdrlen; 2144 hdr_old.bh_caplen = caplen; 2145 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old, 2146 sizeof(hdr_old)); 2147 goto copy; 2148 } 2149#endif 2150 2151 /* 2152 * Append the bpf header. Note we append the actual header size, but 2153 * move forward the length of the header plus padding. 2154 */ 2155 bzero(&hdr, sizeof(hdr)); 2156 if (do_timestamp) 2157 bpf_bintime2ts(bt, &hdr.bh_tstamp, tstype); 2158 hdr.bh_datalen = pktlen; 2159 hdr.bh_hdrlen = hdrlen; 2160 hdr.bh_caplen = caplen; 2161 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr)); 2162 2163 /* 2164 * Copy the packet data into the store buffer and update its length. 2165 */ 2166#ifndef BURN_BRIDGES 2167copy: 2168#endif 2169 (*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen); 2170 d->bd_slen = curlen + totlen; 2171 2172 if (do_wakeup) 2173 bpf_wakeup(d); 2174} 2175 2176/* 2177 * Free buffers currently in use by a descriptor. 2178 * Called on close. 2179 */ 2180static void 2181bpf_freed(struct bpf_d *d) 2182{ 2183 2184 /* 2185 * We don't need to lock out interrupts since this descriptor has 2186 * been detached from its interface and it yet hasn't been marked 2187 * free. 2188 */ 2189 bpf_free(d); 2190 if (d->bd_rfilter != NULL) { 2191 free((caddr_t)d->bd_rfilter, M_BPF); 2192#ifdef BPF_JITTER 2193 if (d->bd_bfilter != NULL) 2194 bpf_destroy_jit_filter(d->bd_bfilter); 2195#endif 2196 } 2197 if (d->bd_wfilter != NULL) 2198 free((caddr_t)d->bd_wfilter, M_BPF); 2199 mtx_destroy(&d->bd_mtx); 2200} 2201 2202/* 2203 * Attach an interface to bpf. dlt is the link layer type; hdrlen is the 2204 * fixed size of the link header (variable length headers not yet supported). 2205 */ 2206void 2207bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2208{ 2209 2210 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); 2211} 2212 2213/* 2214 * Attach an interface to bpf. ifp is a pointer to the structure 2215 * defining the interface to be attached, dlt is the link layer type, 2216 * and hdrlen is the fixed size of the link header (variable length 2217 * headers are not yet supporrted). 2218 */ 2219void 2220bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2221{ 2222 struct bpf_if *bp; 2223 2224 bp = malloc(sizeof(*bp), M_BPF, M_NOWAIT | M_ZERO); 2225 if (bp == NULL) 2226 panic("bpfattach"); 2227 2228 LIST_INIT(&bp->bif_dlist); 2229 bp->bif_ifp = ifp; 2230 bp->bif_dlt = dlt; 2231 mtx_init(&bp->bif_mtx, "bpf interface lock", NULL, MTX_DEF); 2232 KASSERT(*driverp == NULL, ("bpfattach2: driverp already initialized")); 2233 *driverp = bp; 2234 2235 mtx_lock(&bpf_mtx); 2236 LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next); 2237 mtx_unlock(&bpf_mtx); 2238 2239 bp->bif_hdrlen = hdrlen; 2240 2241 if (bootverbose) 2242 if_printf(ifp, "bpf attached\n"); 2243} 2244 2245/* 2246 * Detach bpf from an interface. This involves detaching each descriptor 2247 * associated with the interface, and leaving bd_bif NULL. Notify each 2248 * descriptor as it's detached so that any sleepers wake up and get 2249 * ENXIO. 2250 */ 2251void 2252bpfdetach(struct ifnet *ifp) 2253{ 2254 struct bpf_if *bp; 2255 struct bpf_d *d; 2256 2257 /* Locate BPF interface information */ 2258 mtx_lock(&bpf_mtx); 2259 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2260 if (ifp == bp->bif_ifp) 2261 break; 2262 } 2263 2264 /* Interface wasn't attached */ 2265 if ((bp == NULL) || (bp->bif_ifp == NULL)) { 2266 mtx_unlock(&bpf_mtx); 2267 printf("bpfdetach: %s was not attached\n", ifp->if_xname); 2268 return; 2269 } 2270 2271 LIST_REMOVE(bp, bif_next); 2272 mtx_unlock(&bpf_mtx); 2273 2274 while ((d = LIST_FIRST(&bp->bif_dlist)) != NULL) { 2275 bpf_detachd(d); 2276 BPFD_LOCK(d); 2277 bpf_wakeup(d); 2278 BPFD_UNLOCK(d); 2279 } 2280 2281 mtx_destroy(&bp->bif_mtx); 2282 free(bp, M_BPF); 2283} 2284 2285/* 2286 * Get a list of available data link type of the interface. 2287 */ 2288static int 2289bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2290{ 2291 int n, error; 2292 struct ifnet *ifp; 2293 struct bpf_if *bp; 2294 2295 ifp = d->bd_bif->bif_ifp; 2296 n = 0; 2297 error = 0; 2298 mtx_lock(&bpf_mtx); 2299 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2300 if (bp->bif_ifp != ifp) 2301 continue; 2302 if (bfl->bfl_list != NULL) { 2303 if (n >= bfl->bfl_len) { 2304 mtx_unlock(&bpf_mtx); 2305 return (ENOMEM); 2306 } 2307 error = copyout(&bp->bif_dlt, 2308 bfl->bfl_list + n, sizeof(u_int)); 2309 } 2310 n++; 2311 } 2312 mtx_unlock(&bpf_mtx); 2313 bfl->bfl_len = n; 2314 return (error); 2315} 2316 2317/* 2318 * Set the data link type of a BPF instance. 2319 */ 2320static int 2321bpf_setdlt(struct bpf_d *d, u_int dlt) 2322{ 2323 int error, opromisc; 2324 struct ifnet *ifp; 2325 struct bpf_if *bp; 2326 2327 if (d->bd_bif->bif_dlt == dlt) 2328 return (0); 2329 ifp = d->bd_bif->bif_ifp; 2330 mtx_lock(&bpf_mtx); 2331 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2332 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2333 break; 2334 } 2335 mtx_unlock(&bpf_mtx); 2336 if (bp != NULL) { 2337 opromisc = d->bd_promisc; 2338 bpf_detachd(d); 2339 bpf_attachd(d, bp); 2340 BPFD_LOCK(d); 2341 reset_d(d); 2342 BPFD_UNLOCK(d); 2343 if (opromisc) { 2344 error = ifpromisc(bp->bif_ifp, 1); 2345 if (error) 2346 if_printf(bp->bif_ifp, 2347 "bpf_setdlt: ifpromisc failed (%d)\n", 2348 error); 2349 else 2350 d->bd_promisc = 1; 2351 } 2352 } 2353 return (bp == NULL ? EINVAL : 0); 2354} 2355 2356static void 2357bpf_drvinit(void *unused) 2358{ 2359 struct cdev *dev; 2360 2361 mtx_init(&bpf_mtx, "bpf global lock", NULL, MTX_DEF); 2362 LIST_INIT(&bpf_iflist); 2363 2364 dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf"); 2365 /* For compatibility */ 2366 make_dev_alias(dev, "bpf0"); 2367} 2368 2369/* 2370 * Zero out the various packet counters associated with all of the bpf 2371 * descriptors. At some point, we will probably want to get a bit more 2372 * granular and allow the user to specify descriptors to be zeroed. 2373 */ 2374static void 2375bpf_zero_counters(void) 2376{ 2377 struct bpf_if *bp; 2378 struct bpf_d *bd; 2379 2380 mtx_lock(&bpf_mtx); 2381 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2382 BPFIF_LOCK(bp); 2383 LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { 2384 BPFD_LOCK(bd); 2385 bd->bd_rcount = 0; 2386 bd->bd_dcount = 0; 2387 bd->bd_fcount = 0; 2388 bd->bd_wcount = 0; 2389 bd->bd_wfcount = 0; 2390 bd->bd_zcopy = 0; 2391 BPFD_UNLOCK(bd); 2392 } 2393 BPFIF_UNLOCK(bp); 2394 } 2395 mtx_unlock(&bpf_mtx); 2396} 2397 2398static void 2399bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) 2400{ 2401 2402 bzero(d, sizeof(*d)); 2403 BPFD_LOCK_ASSERT(bd); 2404 d->bd_structsize = sizeof(*d); 2405 d->bd_immediate = bd->bd_immediate; 2406 d->bd_promisc = bd->bd_promisc; 2407 d->bd_hdrcmplt = bd->bd_hdrcmplt; 2408 d->bd_direction = bd->bd_direction; 2409 d->bd_feedback = bd->bd_feedback; 2410 d->bd_async = bd->bd_async; 2411 d->bd_rcount = bd->bd_rcount; 2412 d->bd_dcount = bd->bd_dcount; 2413 d->bd_fcount = bd->bd_fcount; 2414 d->bd_sig = bd->bd_sig; 2415 d->bd_slen = bd->bd_slen; 2416 d->bd_hlen = bd->bd_hlen; 2417 d->bd_bufsize = bd->bd_bufsize; 2418 d->bd_pid = bd->bd_pid; 2419 strlcpy(d->bd_ifname, 2420 bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ); 2421 d->bd_locked = bd->bd_locked; 2422 d->bd_wcount = bd->bd_wcount; 2423 d->bd_wdcount = bd->bd_wdcount; 2424 d->bd_wfcount = bd->bd_wfcount; 2425 d->bd_zcopy = bd->bd_zcopy; 2426 d->bd_bufmode = bd->bd_bufmode; 2427} 2428 2429static int 2430bpf_stats_sysctl(SYSCTL_HANDLER_ARGS) 2431{ 2432 struct xbpf_d *xbdbuf, *xbd, zerostats; 2433 int index, error; 2434 struct bpf_if *bp; 2435 struct bpf_d *bd; 2436 2437 /* 2438 * XXX This is not technically correct. It is possible for non 2439 * privileged users to open bpf devices. It would make sense 2440 * if the users who opened the devices were able to retrieve 2441 * the statistics for them, too. 2442 */ 2443 error = priv_check(req->td, PRIV_NET_BPF); 2444 if (error) 2445 return (error); 2446 /* 2447 * Check to see if the user is requesting that the counters be 2448 * zeroed out. Explicitly check that the supplied data is zeroed, 2449 * as we aren't allowing the user to set the counters currently. 2450 */ 2451 if (req->newptr != NULL) { 2452 if (req->newlen != sizeof(zerostats)) 2453 return (EINVAL); 2454 bzero(&zerostats, sizeof(zerostats)); 2455 xbd = req->newptr; 2456 if (bcmp(xbd, &zerostats, sizeof(*xbd)) != 0) 2457 return (EINVAL); 2458 bpf_zero_counters(); 2459 return (0); 2460 } 2461 if (req->oldptr == NULL) 2462 return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd))); 2463 if (bpf_bpfd_cnt == 0) 2464 return (SYSCTL_OUT(req, 0, 0)); 2465 xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK); 2466 mtx_lock(&bpf_mtx); 2467 if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) { 2468 mtx_unlock(&bpf_mtx); 2469 free(xbdbuf, M_BPF); 2470 return (ENOMEM); 2471 } 2472 index = 0; 2473 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2474 BPFIF_LOCK(bp); 2475 LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { 2476 xbd = &xbdbuf[index++]; 2477 BPFD_LOCK(bd); 2478 bpfstats_fill_xbpf(xbd, bd); 2479 BPFD_UNLOCK(bd); 2480 } 2481 BPFIF_UNLOCK(bp); 2482 } 2483 mtx_unlock(&bpf_mtx); 2484 error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd)); 2485 free(xbdbuf, M_BPF); 2486 return (error); 2487} 2488 2489SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL); 2490 2491#else /* !DEV_BPF && !NETGRAPH_BPF */ 2492/* 2493 * NOP stubs to allow bpf-using drivers to load and function. 2494 * 2495 * A 'better' implementation would allow the core bpf functionality 2496 * to be loaded at runtime. 2497 */ 2498static struct bpf_if bp_null; 2499 2500void 2501bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 2502{ 2503} 2504 2505void 2506bpf_mtap(struct bpf_if *bp, struct mbuf *m) 2507{ 2508} 2509 2510void 2511bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m) 2512{ 2513} 2514 2515void 2516bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2517{ 2518 2519 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); 2520} 2521 2522void 2523bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2524{ 2525 2526 *driverp = &bp_null; 2527} 2528 2529void 2530bpfdetach(struct ifnet *ifp) 2531{ 2532} 2533 2534u_int 2535bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) 2536{ 2537 return -1; /* "no filter" behaviour */ 2538} 2539 2540int 2541bpf_validate(const struct bpf_insn *f, int len) 2542{ 2543 return 0; /* false */ 2544} 2545 2546#endif /* !DEV_BPF && !NETGRAPH_BPF */ 2547