bpf.c revision 185348
1/*- 2 * Copyright (c) 1990, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from the Stanford/CMU enet packet filter, 6 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 7 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 8 * Berkeley Laboratory. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 35 */ 36 37#include <sys/cdefs.h> 38__FBSDID("$FreeBSD: head/sys/net/bpf.c 185348 2008-11-26 22:32:07Z zec $"); 39 40#include "opt_bpf.h" 41#include "opt_mac.h" 42#include "opt_netgraph.h" 43 44#include <sys/types.h> 45#include <sys/param.h> 46#include <sys/systm.h> 47#include <sys/conf.h> 48#include <sys/fcntl.h> 49#include <sys/malloc.h> 50#include <sys/mbuf.h> 51#include <sys/time.h> 52#include <sys/priv.h> 53#include <sys/proc.h> 54#include <sys/signalvar.h> 55#include <sys/filio.h> 56#include <sys/sockio.h> 57#include <sys/ttycom.h> 58#include <sys/uio.h> 59#include <sys/vimage.h> 60 61#include <sys/event.h> 62#include <sys/file.h> 63#include <sys/poll.h> 64#include <sys/proc.h> 65 66#include <sys/socket.h> 67 68#include <net/if.h> 69#include <net/bpf.h> 70#include <net/bpf_buffer.h> 71#ifdef BPF_JITTER 72#include <net/bpf_jitter.h> 73#endif 74#include <net/bpf_zerocopy.h> 75#include <net/bpfdesc.h> 76 77#include <netinet/in.h> 78#include <netinet/if_ether.h> 79#include <sys/kernel.h> 80#include <sys/sysctl.h> 81 82#include <net80211/ieee80211_freebsd.h> 83 84#include <security/mac/mac_framework.h> 85 86MALLOC_DEFINE(M_BPF, "BPF", "BPF data"); 87 88#if defined(DEV_BPF) || defined(NETGRAPH_BPF) 89 90#define PRINET 26 /* interruptible */ 91 92/* 93 * bpf_iflist is a list of BPF interface structures, each corresponding to a 94 * specific DLT. The same network interface might have several BPF interface 95 * structures registered by different layers in the stack (i.e., 802.11 96 * frames, ethernet frames, etc). 97 */ 98static LIST_HEAD(, bpf_if) bpf_iflist; 99static struct mtx bpf_mtx; /* bpf global lock */ 100static int bpf_bpfd_cnt; 101 102static void bpf_attachd(struct bpf_d *, struct bpf_if *); 103static void bpf_detachd(struct bpf_d *); 104static void bpf_freed(struct bpf_d *); 105static int bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **, 106 struct sockaddr *, int *, struct bpf_insn *); 107static int bpf_setif(struct bpf_d *, struct ifreq *); 108static void bpf_timed_out(void *); 109static __inline void 110 bpf_wakeup(struct bpf_d *); 111static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 112 void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int), 113 struct timeval *); 114static void reset_d(struct bpf_d *); 115static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd); 116static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 117static int bpf_setdlt(struct bpf_d *, u_int); 118static void filt_bpfdetach(struct knote *); 119static int filt_bpfread(struct knote *, long); 120static void bpf_drvinit(void *); 121static int bpf_stats_sysctl(SYSCTL_HANDLER_ARGS); 122 123SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW, 0, "bpf sysctl"); 124int bpf_maxinsns = BPF_MAXINSNS; 125SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW, 126 &bpf_maxinsns, 0, "Maximum bpf program instructions"); 127static int bpf_zerocopy_enable = 0; 128SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW, 129 &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions"); 130SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_RW, 131 bpf_stats_sysctl, "bpf statistics portal"); 132 133static d_open_t bpfopen; 134static d_read_t bpfread; 135static d_write_t bpfwrite; 136static d_ioctl_t bpfioctl; 137static d_poll_t bpfpoll; 138static d_kqfilter_t bpfkqfilter; 139 140static struct cdevsw bpf_cdevsw = { 141 .d_version = D_VERSION, 142 .d_open = bpfopen, 143 .d_read = bpfread, 144 .d_write = bpfwrite, 145 .d_ioctl = bpfioctl, 146 .d_poll = bpfpoll, 147 .d_name = "bpf", 148 .d_kqfilter = bpfkqfilter, 149}; 150 151static struct filterops bpfread_filtops = 152 { 1, NULL, filt_bpfdetach, filt_bpfread }; 153 154/* 155 * Wrapper functions for various buffering methods. If the set of buffer 156 * modes expands, we will probably want to introduce a switch data structure 157 * similar to protosw, et. 158 */ 159static void 160bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, 161 u_int len) 162{ 163 164 BPFD_LOCK_ASSERT(d); 165 166 switch (d->bd_bufmode) { 167 case BPF_BUFMODE_BUFFER: 168 return (bpf_buffer_append_bytes(d, buf, offset, src, len)); 169 170 case BPF_BUFMODE_ZBUF: 171 d->bd_zcopy++; 172 return (bpf_zerocopy_append_bytes(d, buf, offset, src, len)); 173 174 default: 175 panic("bpf_buf_append_bytes"); 176 } 177} 178 179static void 180bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src, 181 u_int len) 182{ 183 184 BPFD_LOCK_ASSERT(d); 185 186 switch (d->bd_bufmode) { 187 case BPF_BUFMODE_BUFFER: 188 return (bpf_buffer_append_mbuf(d, buf, offset, src, len)); 189 190 case BPF_BUFMODE_ZBUF: 191 d->bd_zcopy++; 192 return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len)); 193 194 default: 195 panic("bpf_buf_append_mbuf"); 196 } 197} 198 199/* 200 * This function gets called when the free buffer is re-assigned. 201 */ 202static void 203bpf_buf_reclaimed(struct bpf_d *d) 204{ 205 206 BPFD_LOCK_ASSERT(d); 207 208 switch (d->bd_bufmode) { 209 case BPF_BUFMODE_BUFFER: 210 return; 211 212 case BPF_BUFMODE_ZBUF: 213 bpf_zerocopy_buf_reclaimed(d); 214 return; 215 216 default: 217 panic("bpf_buf_reclaimed"); 218 } 219} 220 221/* 222 * If the buffer mechanism has a way to decide that a held buffer can be made 223 * free, then it is exposed via the bpf_canfreebuf() interface. (1) is 224 * returned if the buffer can be discarded, (0) is returned if it cannot. 225 */ 226static int 227bpf_canfreebuf(struct bpf_d *d) 228{ 229 230 BPFD_LOCK_ASSERT(d); 231 232 switch (d->bd_bufmode) { 233 case BPF_BUFMODE_ZBUF: 234 return (bpf_zerocopy_canfreebuf(d)); 235 } 236 return (0); 237} 238 239/* 240 * Allow the buffer model to indicate that the current store buffer is 241 * immutable, regardless of the appearance of space. Return (1) if the 242 * buffer is writable, and (0) if not. 243 */ 244static int 245bpf_canwritebuf(struct bpf_d *d) 246{ 247 248 BPFD_LOCK_ASSERT(d); 249 250 switch (d->bd_bufmode) { 251 case BPF_BUFMODE_ZBUF: 252 return (bpf_zerocopy_canwritebuf(d)); 253 } 254 return (1); 255} 256 257/* 258 * Notify buffer model that an attempt to write to the store buffer has 259 * resulted in a dropped packet, in which case the buffer may be considered 260 * full. 261 */ 262static void 263bpf_buffull(struct bpf_d *d) 264{ 265 266 BPFD_LOCK_ASSERT(d); 267 268 switch (d->bd_bufmode) { 269 case BPF_BUFMODE_ZBUF: 270 bpf_zerocopy_buffull(d); 271 break; 272 } 273} 274 275/* 276 * Notify the buffer model that a buffer has moved into the hold position. 277 */ 278void 279bpf_bufheld(struct bpf_d *d) 280{ 281 282 BPFD_LOCK_ASSERT(d); 283 284 switch (d->bd_bufmode) { 285 case BPF_BUFMODE_ZBUF: 286 bpf_zerocopy_bufheld(d); 287 break; 288 } 289} 290 291static void 292bpf_free(struct bpf_d *d) 293{ 294 295 switch (d->bd_bufmode) { 296 case BPF_BUFMODE_BUFFER: 297 return (bpf_buffer_free(d)); 298 299 case BPF_BUFMODE_ZBUF: 300 return (bpf_zerocopy_free(d)); 301 302 default: 303 panic("bpf_buf_free"); 304 } 305} 306 307static int 308bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio) 309{ 310 311 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) 312 return (EOPNOTSUPP); 313 return (bpf_buffer_uiomove(d, buf, len, uio)); 314} 315 316static int 317bpf_ioctl_sblen(struct bpf_d *d, u_int *i) 318{ 319 320 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) 321 return (EOPNOTSUPP); 322 return (bpf_buffer_ioctl_sblen(d, i)); 323} 324 325static int 326bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i) 327{ 328 329 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 330 return (EOPNOTSUPP); 331 return (bpf_zerocopy_ioctl_getzmax(td, d, i)); 332} 333 334static int 335bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) 336{ 337 338 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 339 return (EOPNOTSUPP); 340 return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz)); 341} 342 343static int 344bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) 345{ 346 347 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 348 return (EOPNOTSUPP); 349 return (bpf_zerocopy_ioctl_setzbuf(td, d, bz)); 350} 351 352/* 353 * General BPF functions. 354 */ 355static int 356bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp, 357 struct sockaddr *sockp, int *hdrlen, struct bpf_insn *wfilter) 358{ 359 const struct ieee80211_bpf_params *p; 360 struct ether_header *eh; 361 struct mbuf *m; 362 int error; 363 int len; 364 int hlen; 365 int slen; 366 367 /* 368 * Build a sockaddr based on the data link layer type. 369 * We do this at this level because the ethernet header 370 * is copied directly into the data field of the sockaddr. 371 * In the case of SLIP, there is no header and the packet 372 * is forwarded as is. 373 * Also, we are careful to leave room at the front of the mbuf 374 * for the link level header. 375 */ 376 switch (linktype) { 377 378 case DLT_SLIP: 379 sockp->sa_family = AF_INET; 380 hlen = 0; 381 break; 382 383 case DLT_EN10MB: 384 sockp->sa_family = AF_UNSPEC; 385 /* XXX Would MAXLINKHDR be better? */ 386 hlen = ETHER_HDR_LEN; 387 break; 388 389 case DLT_FDDI: 390 sockp->sa_family = AF_IMPLINK; 391 hlen = 0; 392 break; 393 394 case DLT_RAW: 395 sockp->sa_family = AF_UNSPEC; 396 hlen = 0; 397 break; 398 399 case DLT_NULL: 400 /* 401 * null interface types require a 4 byte pseudo header which 402 * corresponds to the address family of the packet. 403 */ 404 sockp->sa_family = AF_UNSPEC; 405 hlen = 4; 406 break; 407 408 case DLT_ATM_RFC1483: 409 /* 410 * en atm driver requires 4-byte atm pseudo header. 411 * though it isn't standard, vpi:vci needs to be 412 * specified anyway. 413 */ 414 sockp->sa_family = AF_UNSPEC; 415 hlen = 12; /* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */ 416 break; 417 418 case DLT_PPP: 419 sockp->sa_family = AF_UNSPEC; 420 hlen = 4; /* This should match PPP_HDRLEN */ 421 break; 422 423 case DLT_IEEE802_11: /* IEEE 802.11 wireless */ 424 sockp->sa_family = AF_IEEE80211; 425 hlen = 0; 426 break; 427 428 case DLT_IEEE802_11_RADIO: /* IEEE 802.11 wireless w/ phy params */ 429 sockp->sa_family = AF_IEEE80211; 430 sockp->sa_len = 12; /* XXX != 0 */ 431 hlen = sizeof(struct ieee80211_bpf_params); 432 break; 433 434 default: 435 return (EIO); 436 } 437 438 len = uio->uio_resid; 439 440 if (len - hlen > ifp->if_mtu) 441 return (EMSGSIZE); 442 443 if ((unsigned)len > MJUM16BYTES) 444 return (EIO); 445 446 if (len <= MHLEN) 447 MGETHDR(m, M_WAIT, MT_DATA); 448 else if (len <= MCLBYTES) 449 m = m_getcl(M_WAIT, MT_DATA, M_PKTHDR); 450 else 451 m = m_getjcl(M_WAIT, MT_DATA, M_PKTHDR, 452#if (MJUMPAGESIZE > MCLBYTES) 453 len <= MJUMPAGESIZE ? MJUMPAGESIZE : 454#endif 455 (len <= MJUM9BYTES ? MJUM9BYTES : MJUM16BYTES)); 456 m->m_pkthdr.len = m->m_len = len; 457 m->m_pkthdr.rcvif = NULL; 458 *mp = m; 459 460 if (m->m_len < hlen) { 461 error = EPERM; 462 goto bad; 463 } 464 465 error = uiomove(mtod(m, u_char *), len, uio); 466 if (error) 467 goto bad; 468 469 slen = bpf_filter(wfilter, mtod(m, u_char *), len, len); 470 if (slen == 0) { 471 error = EPERM; 472 goto bad; 473 } 474 475 /* Check for multicast destination */ 476 switch (linktype) { 477 case DLT_EN10MB: 478 eh = mtod(m, struct ether_header *); 479 if (ETHER_IS_MULTICAST(eh->ether_dhost)) { 480 if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost, 481 ETHER_ADDR_LEN) == 0) 482 m->m_flags |= M_BCAST; 483 else 484 m->m_flags |= M_MCAST; 485 } 486 break; 487 } 488 489 /* 490 * Make room for link header, and copy it to sockaddr 491 */ 492 if (hlen != 0) { 493 if (sockp->sa_family == AF_IEEE80211) { 494 /* 495 * Collect true length from the parameter header 496 * NB: sockp is known to be zero'd so if we do a 497 * short copy unspecified parameters will be 498 * zero. 499 * NB: packet may not be aligned after stripping 500 * bpf params 501 * XXX check ibp_vers 502 */ 503 p = mtod(m, const struct ieee80211_bpf_params *); 504 hlen = p->ibp_len; 505 if (hlen > sizeof(sockp->sa_data)) { 506 error = EINVAL; 507 goto bad; 508 } 509 } 510 bcopy(m->m_data, sockp->sa_data, hlen); 511 } 512 *hdrlen = hlen; 513 514 return (0); 515bad: 516 m_freem(m); 517 return (error); 518} 519 520/* 521 * Attach file to the bpf interface, i.e. make d listen on bp. 522 */ 523static void 524bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 525{ 526 /* 527 * Point d at bp, and add d to the interface's list of listeners. 528 * Finally, point the driver's bpf cookie at the interface so 529 * it will divert packets to bpf. 530 */ 531 BPFIF_LOCK(bp); 532 d->bd_bif = bp; 533 LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); 534 535 bpf_bpfd_cnt++; 536 BPFIF_UNLOCK(bp); 537} 538 539/* 540 * Detach a file from its interface. 541 */ 542static void 543bpf_detachd(struct bpf_d *d) 544{ 545 int error; 546 struct bpf_if *bp; 547 struct ifnet *ifp; 548 549 bp = d->bd_bif; 550 BPFIF_LOCK(bp); 551 BPFD_LOCK(d); 552 ifp = d->bd_bif->bif_ifp; 553 554 /* 555 * Remove d from the interface's descriptor list. 556 */ 557 LIST_REMOVE(d, bd_next); 558 559 bpf_bpfd_cnt--; 560 d->bd_bif = NULL; 561 BPFD_UNLOCK(d); 562 BPFIF_UNLOCK(bp); 563 564 /* 565 * Check if this descriptor had requested promiscuous mode. 566 * If so, turn it off. 567 */ 568 if (d->bd_promisc) { 569 d->bd_promisc = 0; 570 CURVNET_SET(ifp->if_vnet); 571 error = ifpromisc(ifp, 0); 572 CURVNET_RESTORE(); 573 if (error != 0 && error != ENXIO) { 574 /* 575 * ENXIO can happen if a pccard is unplugged 576 * Something is really wrong if we were able to put 577 * the driver into promiscuous mode, but can't 578 * take it out. 579 */ 580 if_printf(bp->bif_ifp, 581 "bpf_detach: ifpromisc failed (%d)\n", error); 582 } 583 } 584} 585 586/* 587 * Close the descriptor by detaching it from its interface, 588 * deallocating its buffers, and marking it free. 589 */ 590static void 591bpf_dtor(void *data) 592{ 593 struct bpf_d *d = data; 594 595 BPFD_LOCK(d); 596 if (d->bd_state == BPF_WAITING) 597 callout_stop(&d->bd_callout); 598 d->bd_state = BPF_IDLE; 599 BPFD_UNLOCK(d); 600 funsetown(&d->bd_sigio); 601 mtx_lock(&bpf_mtx); 602 if (d->bd_bif) 603 bpf_detachd(d); 604 mtx_unlock(&bpf_mtx); 605 selwakeuppri(&d->bd_sel, PRINET); 606#ifdef MAC 607 mac_bpfdesc_destroy(d); 608#endif /* MAC */ 609 knlist_destroy(&d->bd_sel.si_note); 610 bpf_freed(d); 611 free(d, M_BPF); 612} 613 614/* 615 * Open ethernet device. Returns ENXIO for illegal minor device number, 616 * EBUSY if file is open by another process. 617 */ 618/* ARGSUSED */ 619static int 620bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td) 621{ 622 struct bpf_d *d; 623 int error; 624 625 d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO); 626 error = devfs_set_cdevpriv(d, bpf_dtor); 627 if (error != 0) { 628 free(d, M_BPF); 629 return (error); 630 } 631 632 /* 633 * For historical reasons, perform a one-time initialization call to 634 * the buffer routines, even though we're not yet committed to a 635 * particular buffer method. 636 */ 637 bpf_buffer_init(d); 638 d->bd_bufmode = BPF_BUFMODE_BUFFER; 639 d->bd_sig = SIGIO; 640 d->bd_direction = BPF_D_INOUT; 641 d->bd_pid = td->td_proc->p_pid; 642#ifdef MAC 643 mac_bpfdesc_init(d); 644 mac_bpfdesc_create(td->td_ucred, d); 645#endif 646 mtx_init(&d->bd_mtx, devtoname(dev), "bpf cdev lock", MTX_DEF); 647 callout_init(&d->bd_callout, CALLOUT_MPSAFE); 648 knlist_init(&d->bd_sel.si_note, &d->bd_mtx, NULL, NULL, NULL); 649 650 return (0); 651} 652 653/* 654 * bpfread - read next chunk of packets from buffers 655 */ 656static int 657bpfread(struct cdev *dev, struct uio *uio, int ioflag) 658{ 659 struct bpf_d *d; 660 int timed_out; 661 int error; 662 663 error = devfs_get_cdevpriv((void **)&d); 664 if (error != 0) 665 return (error); 666 667 /* 668 * Restrict application to use a buffer the same size as 669 * as kernel buffers. 670 */ 671 if (uio->uio_resid != d->bd_bufsize) 672 return (EINVAL); 673 674 BPFD_LOCK(d); 675 d->bd_pid = curthread->td_proc->p_pid; 676 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) { 677 BPFD_UNLOCK(d); 678 return (EOPNOTSUPP); 679 } 680 if (d->bd_state == BPF_WAITING) 681 callout_stop(&d->bd_callout); 682 timed_out = (d->bd_state == BPF_TIMED_OUT); 683 d->bd_state = BPF_IDLE; 684 /* 685 * If the hold buffer is empty, then do a timed sleep, which 686 * ends when the timeout expires or when enough packets 687 * have arrived to fill the store buffer. 688 */ 689 while (d->bd_hbuf == NULL) { 690 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 691 /* 692 * A packet(s) either arrived since the previous 693 * read or arrived while we were asleep. 694 * Rotate the buffers and return what's here. 695 */ 696 ROTATE_BUFFERS(d); 697 break; 698 } 699 700 /* 701 * No data is available, check to see if the bpf device 702 * is still pointed at a real interface. If not, return 703 * ENXIO so that the userland process knows to rebind 704 * it before using it again. 705 */ 706 if (d->bd_bif == NULL) { 707 BPFD_UNLOCK(d); 708 return (ENXIO); 709 } 710 711 if (ioflag & O_NONBLOCK) { 712 BPFD_UNLOCK(d); 713 return (EWOULDBLOCK); 714 } 715 error = msleep(d, &d->bd_mtx, PRINET|PCATCH, 716 "bpf", d->bd_rtout); 717 if (error == EINTR || error == ERESTART) { 718 BPFD_UNLOCK(d); 719 return (error); 720 } 721 if (error == EWOULDBLOCK) { 722 /* 723 * On a timeout, return what's in the buffer, 724 * which may be nothing. If there is something 725 * in the store buffer, we can rotate the buffers. 726 */ 727 if (d->bd_hbuf) 728 /* 729 * We filled up the buffer in between 730 * getting the timeout and arriving 731 * here, so we don't need to rotate. 732 */ 733 break; 734 735 if (d->bd_slen == 0) { 736 BPFD_UNLOCK(d); 737 return (0); 738 } 739 ROTATE_BUFFERS(d); 740 break; 741 } 742 } 743 /* 744 * At this point, we know we have something in the hold slot. 745 */ 746 BPFD_UNLOCK(d); 747 748 /* 749 * Move data from hold buffer into user space. 750 * We know the entire buffer is transferred since 751 * we checked above that the read buffer is bpf_bufsize bytes. 752 * 753 * XXXRW: More synchronization needed here: what if a second thread 754 * issues a read on the same fd at the same time? Don't want this 755 * getting invalidated. 756 */ 757 error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio); 758 759 BPFD_LOCK(d); 760 d->bd_fbuf = d->bd_hbuf; 761 d->bd_hbuf = NULL; 762 d->bd_hlen = 0; 763 bpf_buf_reclaimed(d); 764 BPFD_UNLOCK(d); 765 766 return (error); 767} 768 769/* 770 * If there are processes sleeping on this descriptor, wake them up. 771 */ 772static __inline void 773bpf_wakeup(struct bpf_d *d) 774{ 775 776 BPFD_LOCK_ASSERT(d); 777 if (d->bd_state == BPF_WAITING) { 778 callout_stop(&d->bd_callout); 779 d->bd_state = BPF_IDLE; 780 } 781 wakeup(d); 782 if (d->bd_async && d->bd_sig && d->bd_sigio) 783 pgsigio(&d->bd_sigio, d->bd_sig, 0); 784 785 selwakeuppri(&d->bd_sel, PRINET); 786 KNOTE_LOCKED(&d->bd_sel.si_note, 0); 787} 788 789static void 790bpf_timed_out(void *arg) 791{ 792 struct bpf_d *d = (struct bpf_d *)arg; 793 794 BPFD_LOCK(d); 795 if (d->bd_state == BPF_WAITING) { 796 d->bd_state = BPF_TIMED_OUT; 797 if (d->bd_slen != 0) 798 bpf_wakeup(d); 799 } 800 BPFD_UNLOCK(d); 801} 802 803static int 804bpf_ready(struct bpf_d *d) 805{ 806 807 BPFD_LOCK_ASSERT(d); 808 809 if (!bpf_canfreebuf(d) && d->bd_hlen != 0) 810 return (1); 811 if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 812 d->bd_slen != 0) 813 return (1); 814 return (0); 815} 816 817static int 818bpfwrite(struct cdev *dev, struct uio *uio, int ioflag) 819{ 820 struct bpf_d *d; 821 struct ifnet *ifp; 822 struct mbuf *m, *mc; 823 struct sockaddr dst; 824 int error, hlen; 825 826 error = devfs_get_cdevpriv((void **)&d); 827 if (error != 0) 828 return (error); 829 830 d->bd_pid = curthread->td_proc->p_pid; 831 d->bd_wcount++; 832 if (d->bd_bif == NULL) { 833 d->bd_wdcount++; 834 return (ENXIO); 835 } 836 837 ifp = d->bd_bif->bif_ifp; 838 839 if ((ifp->if_flags & IFF_UP) == 0) { 840 d->bd_wdcount++; 841 return (ENETDOWN); 842 } 843 844 if (uio->uio_resid == 0) { 845 d->bd_wdcount++; 846 return (0); 847 } 848 849 bzero(&dst, sizeof(dst)); 850 m = NULL; 851 hlen = 0; 852 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp, 853 &m, &dst, &hlen, d->bd_wfilter); 854 if (error) { 855 d->bd_wdcount++; 856 return (error); 857 } 858 d->bd_wfcount++; 859 if (d->bd_hdrcmplt) 860 dst.sa_family = pseudo_AF_HDRCMPLT; 861 862 if (d->bd_feedback) { 863 mc = m_dup(m, M_DONTWAIT); 864 if (mc != NULL) 865 mc->m_pkthdr.rcvif = ifp; 866 /* Set M_PROMISC for outgoing packets to be discarded. */ 867 if (d->bd_direction == BPF_D_INOUT) 868 m->m_flags |= M_PROMISC; 869 } else 870 mc = NULL; 871 872 m->m_pkthdr.len -= hlen; 873 m->m_len -= hlen; 874 m->m_data += hlen; /* XXX */ 875 876#ifdef MAC 877 BPFD_LOCK(d); 878 CURVNET_SET(ifp->if_vnet); 879 mac_bpfdesc_create_mbuf(d, m); 880 CURVNET_RESTORE(); 881 if (mc != NULL) 882 mac_bpfdesc_create_mbuf(d, mc); 883 BPFD_UNLOCK(d); 884#endif 885 886 error = (*ifp->if_output)(ifp, m, &dst, NULL); 887 if (error) 888 d->bd_wdcount++; 889 890 if (mc != NULL) { 891 if (error == 0) 892 (*ifp->if_input)(ifp, mc); 893 else 894 m_freem(mc); 895 } 896 897 return (error); 898} 899 900/* 901 * Reset a descriptor by flushing its packet buffer and clearing the 902 * receive and drop counts. 903 */ 904static void 905reset_d(struct bpf_d *d) 906{ 907 908 mtx_assert(&d->bd_mtx, MA_OWNED); 909 if (d->bd_hbuf) { 910 /* Free the hold buffer. */ 911 d->bd_fbuf = d->bd_hbuf; 912 d->bd_hbuf = NULL; 913 bpf_buf_reclaimed(d); 914 } 915 d->bd_slen = 0; 916 d->bd_hlen = 0; 917 d->bd_rcount = 0; 918 d->bd_dcount = 0; 919 d->bd_fcount = 0; 920 d->bd_wcount = 0; 921 d->bd_wfcount = 0; 922 d->bd_wdcount = 0; 923 d->bd_zcopy = 0; 924} 925 926/* 927 * FIONREAD Check for read packet available. 928 * SIOCGIFADDR Get interface address - convenient hook to driver. 929 * BIOCGBLEN Get buffer len [for read()]. 930 * BIOCSETF Set read filter. 931 * BIOCSETFNR Set read filter without resetting descriptor. 932 * BIOCSETWF Set write filter. 933 * BIOCFLUSH Flush read packet buffer. 934 * BIOCPROMISC Put interface into promiscuous mode. 935 * BIOCGDLT Get link layer type. 936 * BIOCGETIF Get interface name. 937 * BIOCSETIF Set interface. 938 * BIOCSRTIMEOUT Set read timeout. 939 * BIOCGRTIMEOUT Get read timeout. 940 * BIOCGSTATS Get packet stats. 941 * BIOCIMMEDIATE Set immediate mode. 942 * BIOCVERSION Get filter language version. 943 * BIOCGHDRCMPLT Get "header already complete" flag 944 * BIOCSHDRCMPLT Set "header already complete" flag 945 * BIOCGDIRECTION Get packet direction flag 946 * BIOCSDIRECTION Set packet direction flag 947 * BIOCLOCK Set "locked" flag 948 * BIOCFEEDBACK Set packet feedback mode. 949 * BIOCSETZBUF Set current zero-copy buffer locations. 950 * BIOCGETZMAX Get maximum zero-copy buffer size. 951 * BIOCROTZBUF Force rotation of zero-copy buffer 952 * BIOCSETBUFMODE Set buffer mode. 953 * BIOCGETBUFMODE Get current buffer mode. 954 */ 955/* ARGSUSED */ 956static int 957bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, 958 struct thread *td) 959{ 960 struct bpf_d *d; 961 int error; 962 963 error = devfs_get_cdevpriv((void **)&d); 964 if (error != 0) 965 return (error); 966 967 /* 968 * Refresh PID associated with this descriptor. 969 */ 970 BPFD_LOCK(d); 971 d->bd_pid = td->td_proc->p_pid; 972 if (d->bd_state == BPF_WAITING) 973 callout_stop(&d->bd_callout); 974 d->bd_state = BPF_IDLE; 975 BPFD_UNLOCK(d); 976 977 if (d->bd_locked == 1) { 978 switch (cmd) { 979 case BIOCGBLEN: 980 case BIOCFLUSH: 981 case BIOCGDLT: 982 case BIOCGDLTLIST: 983 case BIOCGETIF: 984 case BIOCGRTIMEOUT: 985 case BIOCGSTATS: 986 case BIOCVERSION: 987 case BIOCGRSIG: 988 case BIOCGHDRCMPLT: 989 case BIOCFEEDBACK: 990 case FIONREAD: 991 case BIOCLOCK: 992 case BIOCSRTIMEOUT: 993 case BIOCIMMEDIATE: 994 case TIOCGPGRP: 995 case BIOCROTZBUF: 996 break; 997 default: 998 return (EPERM); 999 } 1000 } 1001 CURVNET_SET(TD_TO_VNET(td)); 1002 switch (cmd) { 1003 1004 default: 1005 error = EINVAL; 1006 break; 1007 1008 /* 1009 * Check for read packet available. 1010 */ 1011 case FIONREAD: 1012 { 1013 int n; 1014 1015 BPFD_LOCK(d); 1016 n = d->bd_slen; 1017 if (d->bd_hbuf) 1018 n += d->bd_hlen; 1019 BPFD_UNLOCK(d); 1020 1021 *(int *)addr = n; 1022 break; 1023 } 1024 1025 case SIOCGIFADDR: 1026 { 1027 struct ifnet *ifp; 1028 1029 if (d->bd_bif == NULL) 1030 error = EINVAL; 1031 else { 1032 ifp = d->bd_bif->bif_ifp; 1033 error = (*ifp->if_ioctl)(ifp, cmd, addr); 1034 } 1035 break; 1036 } 1037 1038 /* 1039 * Get buffer len [for read()]. 1040 */ 1041 case BIOCGBLEN: 1042 *(u_int *)addr = d->bd_bufsize; 1043 break; 1044 1045 /* 1046 * Set buffer length. 1047 */ 1048 case BIOCSBLEN: 1049 error = bpf_ioctl_sblen(d, (u_int *)addr); 1050 break; 1051 1052 /* 1053 * Set link layer read filter. 1054 */ 1055 case BIOCSETF: 1056 case BIOCSETFNR: 1057 case BIOCSETWF: 1058 error = bpf_setf(d, (struct bpf_program *)addr, cmd); 1059 break; 1060 1061 /* 1062 * Flush read packet buffer. 1063 */ 1064 case BIOCFLUSH: 1065 BPFD_LOCK(d); 1066 reset_d(d); 1067 BPFD_UNLOCK(d); 1068 break; 1069 1070 /* 1071 * Put interface into promiscuous mode. 1072 */ 1073 case BIOCPROMISC: 1074 if (d->bd_bif == NULL) { 1075 /* 1076 * No interface attached yet. 1077 */ 1078 error = EINVAL; 1079 break; 1080 } 1081 if (d->bd_promisc == 0) { 1082 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1083 if (error == 0) 1084 d->bd_promisc = 1; 1085 } 1086 break; 1087 1088 /* 1089 * Get current data link type. 1090 */ 1091 case BIOCGDLT: 1092 if (d->bd_bif == NULL) 1093 error = EINVAL; 1094 else 1095 *(u_int *)addr = d->bd_bif->bif_dlt; 1096 break; 1097 1098 /* 1099 * Get a list of supported data link types. 1100 */ 1101 case BIOCGDLTLIST: 1102 if (d->bd_bif == NULL) 1103 error = EINVAL; 1104 else 1105 error = bpf_getdltlist(d, (struct bpf_dltlist *)addr); 1106 break; 1107 1108 /* 1109 * Set data link type. 1110 */ 1111 case BIOCSDLT: 1112 if (d->bd_bif == NULL) 1113 error = EINVAL; 1114 else 1115 error = bpf_setdlt(d, *(u_int *)addr); 1116 break; 1117 1118 /* 1119 * Get interface name. 1120 */ 1121 case BIOCGETIF: 1122 if (d->bd_bif == NULL) 1123 error = EINVAL; 1124 else { 1125 struct ifnet *const ifp = d->bd_bif->bif_ifp; 1126 struct ifreq *const ifr = (struct ifreq *)addr; 1127 1128 strlcpy(ifr->ifr_name, ifp->if_xname, 1129 sizeof(ifr->ifr_name)); 1130 } 1131 break; 1132 1133 /* 1134 * Set interface. 1135 */ 1136 case BIOCSETIF: 1137 error = bpf_setif(d, (struct ifreq *)addr); 1138 break; 1139 1140 /* 1141 * Set read timeout. 1142 */ 1143 case BIOCSRTIMEOUT: 1144 { 1145 struct timeval *tv = (struct timeval *)addr; 1146 1147 /* 1148 * Subtract 1 tick from tvtohz() since this isn't 1149 * a one-shot timer. 1150 */ 1151 if ((error = itimerfix(tv)) == 0) 1152 d->bd_rtout = tvtohz(tv) - 1; 1153 break; 1154 } 1155 1156 /* 1157 * Get read timeout. 1158 */ 1159 case BIOCGRTIMEOUT: 1160 { 1161 struct timeval *tv = (struct timeval *)addr; 1162 1163 tv->tv_sec = d->bd_rtout / hz; 1164 tv->tv_usec = (d->bd_rtout % hz) * tick; 1165 break; 1166 } 1167 1168 /* 1169 * Get packet stats. 1170 */ 1171 case BIOCGSTATS: 1172 { 1173 struct bpf_stat *bs = (struct bpf_stat *)addr; 1174 1175 /* XXXCSJP overflow */ 1176 bs->bs_recv = d->bd_rcount; 1177 bs->bs_drop = d->bd_dcount; 1178 break; 1179 } 1180 1181 /* 1182 * Set immediate mode. 1183 */ 1184 case BIOCIMMEDIATE: 1185 d->bd_immediate = *(u_int *)addr; 1186 break; 1187 1188 case BIOCVERSION: 1189 { 1190 struct bpf_version *bv = (struct bpf_version *)addr; 1191 1192 bv->bv_major = BPF_MAJOR_VERSION; 1193 bv->bv_minor = BPF_MINOR_VERSION; 1194 break; 1195 } 1196 1197 /* 1198 * Get "header already complete" flag 1199 */ 1200 case BIOCGHDRCMPLT: 1201 *(u_int *)addr = d->bd_hdrcmplt; 1202 break; 1203 1204 /* 1205 * Set "header already complete" flag 1206 */ 1207 case BIOCSHDRCMPLT: 1208 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1209 break; 1210 1211 /* 1212 * Get packet direction flag 1213 */ 1214 case BIOCGDIRECTION: 1215 *(u_int *)addr = d->bd_direction; 1216 break; 1217 1218 /* 1219 * Set packet direction flag 1220 */ 1221 case BIOCSDIRECTION: 1222 { 1223 u_int direction; 1224 1225 direction = *(u_int *)addr; 1226 switch (direction) { 1227 case BPF_D_IN: 1228 case BPF_D_INOUT: 1229 case BPF_D_OUT: 1230 d->bd_direction = direction; 1231 break; 1232 default: 1233 error = EINVAL; 1234 } 1235 } 1236 break; 1237 1238 case BIOCFEEDBACK: 1239 d->bd_feedback = *(u_int *)addr; 1240 break; 1241 1242 case BIOCLOCK: 1243 d->bd_locked = 1; 1244 break; 1245 1246 case FIONBIO: /* Non-blocking I/O */ 1247 break; 1248 1249 case FIOASYNC: /* Send signal on receive packets */ 1250 d->bd_async = *(int *)addr; 1251 break; 1252 1253 case FIOSETOWN: 1254 error = fsetown(*(int *)addr, &d->bd_sigio); 1255 break; 1256 1257 case FIOGETOWN: 1258 *(int *)addr = fgetown(&d->bd_sigio); 1259 break; 1260 1261 /* This is deprecated, FIOSETOWN should be used instead. */ 1262 case TIOCSPGRP: 1263 error = fsetown(-(*(int *)addr), &d->bd_sigio); 1264 break; 1265 1266 /* This is deprecated, FIOGETOWN should be used instead. */ 1267 case TIOCGPGRP: 1268 *(int *)addr = -fgetown(&d->bd_sigio); 1269 break; 1270 1271 case BIOCSRSIG: /* Set receive signal */ 1272 { 1273 u_int sig; 1274 1275 sig = *(u_int *)addr; 1276 1277 if (sig >= NSIG) 1278 error = EINVAL; 1279 else 1280 d->bd_sig = sig; 1281 break; 1282 } 1283 case BIOCGRSIG: 1284 *(u_int *)addr = d->bd_sig; 1285 break; 1286 1287 case BIOCGETBUFMODE: 1288 *(u_int *)addr = d->bd_bufmode; 1289 break; 1290 1291 case BIOCSETBUFMODE: 1292 /* 1293 * Allow the buffering mode to be changed as long as we 1294 * haven't yet committed to a particular mode. Our 1295 * definition of commitment, for now, is whether or not a 1296 * buffer has been allocated or an interface attached, since 1297 * that's the point where things get tricky. 1298 */ 1299 switch (*(u_int *)addr) { 1300 case BPF_BUFMODE_BUFFER: 1301 break; 1302 1303 case BPF_BUFMODE_ZBUF: 1304 if (bpf_zerocopy_enable) 1305 break; 1306 /* FALLSTHROUGH */ 1307 1308 default: 1309 return (EINVAL); 1310 } 1311 1312 BPFD_LOCK(d); 1313 if (d->bd_sbuf != NULL || d->bd_hbuf != NULL || 1314 d->bd_fbuf != NULL || d->bd_bif != NULL) { 1315 BPFD_UNLOCK(d); 1316 return (EBUSY); 1317 } 1318 d->bd_bufmode = *(u_int *)addr; 1319 BPFD_UNLOCK(d); 1320 break; 1321 1322 case BIOCGETZMAX: 1323 return (bpf_ioctl_getzmax(td, d, (size_t *)addr)); 1324 1325 case BIOCSETZBUF: 1326 return (bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr)); 1327 1328 case BIOCROTZBUF: 1329 return (bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr)); 1330 } 1331 CURVNET_RESTORE(); 1332 return (error); 1333} 1334 1335/* 1336 * Set d's packet filter program to fp. If this file already has a filter, 1337 * free it and replace it. Returns EINVAL for bogus requests. 1338 */ 1339static int 1340bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) 1341{ 1342 struct bpf_insn *fcode, *old; 1343 u_int wfilter, flen, size; 1344#ifdef BPF_JITTER 1345 bpf_jit_filter *ofunc; 1346#endif 1347 1348 if (cmd == BIOCSETWF) { 1349 old = d->bd_wfilter; 1350 wfilter = 1; 1351#ifdef BPF_JITTER 1352 ofunc = NULL; 1353#endif 1354 } else { 1355 wfilter = 0; 1356 old = d->bd_rfilter; 1357#ifdef BPF_JITTER 1358 ofunc = d->bd_bfilter; 1359#endif 1360 } 1361 if (fp->bf_insns == NULL) { 1362 if (fp->bf_len != 0) 1363 return (EINVAL); 1364 BPFD_LOCK(d); 1365 if (wfilter) 1366 d->bd_wfilter = NULL; 1367 else { 1368 d->bd_rfilter = NULL; 1369#ifdef BPF_JITTER 1370 d->bd_bfilter = NULL; 1371#endif 1372 if (cmd == BIOCSETF) 1373 reset_d(d); 1374 } 1375 BPFD_UNLOCK(d); 1376 if (old != NULL) 1377 free((caddr_t)old, M_BPF); 1378#ifdef BPF_JITTER 1379 if (ofunc != NULL) 1380 bpf_destroy_jit_filter(ofunc); 1381#endif 1382 return (0); 1383 } 1384 flen = fp->bf_len; 1385 if (flen > bpf_maxinsns) 1386 return (EINVAL); 1387 1388 size = flen * sizeof(*fp->bf_insns); 1389 fcode = (struct bpf_insn *)malloc(size, M_BPF, M_WAITOK); 1390 if (copyin((caddr_t)fp->bf_insns, (caddr_t)fcode, size) == 0 && 1391 bpf_validate(fcode, (int)flen)) { 1392 BPFD_LOCK(d); 1393 if (wfilter) 1394 d->bd_wfilter = fcode; 1395 else { 1396 d->bd_rfilter = fcode; 1397#ifdef BPF_JITTER 1398 d->bd_bfilter = bpf_jitter(fcode, flen); 1399#endif 1400 if (cmd == BIOCSETF) 1401 reset_d(d); 1402 } 1403 BPFD_UNLOCK(d); 1404 if (old != NULL) 1405 free((caddr_t)old, M_BPF); 1406#ifdef BPF_JITTER 1407 if (ofunc != NULL) 1408 bpf_destroy_jit_filter(ofunc); 1409#endif 1410 1411 return (0); 1412 } 1413 free((caddr_t)fcode, M_BPF); 1414 return (EINVAL); 1415} 1416 1417/* 1418 * Detach a file from its current interface (if attached at all) and attach 1419 * to the interface indicated by the name stored in ifr. 1420 * Return an errno or 0. 1421 */ 1422static int 1423bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1424{ 1425 struct bpf_if *bp; 1426 struct ifnet *theywant; 1427 1428 theywant = ifunit(ifr->ifr_name); 1429 if (theywant == NULL || theywant->if_bpf == NULL) 1430 return (ENXIO); 1431 1432 bp = theywant->if_bpf; 1433 1434 /* 1435 * Behavior here depends on the buffering model. If we're using 1436 * kernel memory buffers, then we can allocate them here. If we're 1437 * using zero-copy, then the user process must have registered 1438 * buffers by the time we get here. If not, return an error. 1439 * 1440 * XXXRW: There are locking issues here with multi-threaded use: what 1441 * if two threads try to set the interface at once? 1442 */ 1443 switch (d->bd_bufmode) { 1444 case BPF_BUFMODE_BUFFER: 1445 if (d->bd_sbuf == NULL) 1446 bpf_buffer_alloc(d); 1447 KASSERT(d->bd_sbuf != NULL, ("bpf_setif: bd_sbuf NULL")); 1448 break; 1449 1450 case BPF_BUFMODE_ZBUF: 1451 if (d->bd_sbuf == NULL) 1452 return (EINVAL); 1453 break; 1454 1455 default: 1456 panic("bpf_setif: bufmode %d", d->bd_bufmode); 1457 } 1458 if (bp != d->bd_bif) { 1459 if (d->bd_bif) 1460 /* 1461 * Detach if attached to something else. 1462 */ 1463 bpf_detachd(d); 1464 1465 bpf_attachd(d, bp); 1466 } 1467 BPFD_LOCK(d); 1468 reset_d(d); 1469 BPFD_UNLOCK(d); 1470 return (0); 1471} 1472 1473/* 1474 * Support for select() and poll() system calls 1475 * 1476 * Return true iff the specific operation will not block indefinitely. 1477 * Otherwise, return false but make a note that a selwakeup() must be done. 1478 */ 1479static int 1480bpfpoll(struct cdev *dev, int events, struct thread *td) 1481{ 1482 struct bpf_d *d; 1483 int revents; 1484 1485 if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL) 1486 return (events & 1487 (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM)); 1488 1489 /* 1490 * Refresh PID associated with this descriptor. 1491 */ 1492 revents = events & (POLLOUT | POLLWRNORM); 1493 BPFD_LOCK(d); 1494 d->bd_pid = td->td_proc->p_pid; 1495 if (events & (POLLIN | POLLRDNORM)) { 1496 if (bpf_ready(d)) 1497 revents |= events & (POLLIN | POLLRDNORM); 1498 else { 1499 selrecord(td, &d->bd_sel); 1500 /* Start the read timeout if necessary. */ 1501 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1502 callout_reset(&d->bd_callout, d->bd_rtout, 1503 bpf_timed_out, d); 1504 d->bd_state = BPF_WAITING; 1505 } 1506 } 1507 } 1508 BPFD_UNLOCK(d); 1509 return (revents); 1510} 1511 1512/* 1513 * Support for kevent() system call. Register EVFILT_READ filters and 1514 * reject all others. 1515 */ 1516int 1517bpfkqfilter(struct cdev *dev, struct knote *kn) 1518{ 1519 struct bpf_d *d; 1520 1521 if (devfs_get_cdevpriv((void **)&d) != 0 || 1522 kn->kn_filter != EVFILT_READ) 1523 return (1); 1524 1525 /* 1526 * Refresh PID associated with this descriptor. 1527 */ 1528 BPFD_LOCK(d); 1529 d->bd_pid = curthread->td_proc->p_pid; 1530 kn->kn_fop = &bpfread_filtops; 1531 kn->kn_hook = d; 1532 knlist_add(&d->bd_sel.si_note, kn, 1); 1533 BPFD_UNLOCK(d); 1534 1535 return (0); 1536} 1537 1538static void 1539filt_bpfdetach(struct knote *kn) 1540{ 1541 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; 1542 1543 knlist_remove(&d->bd_sel.si_note, kn, 0); 1544} 1545 1546static int 1547filt_bpfread(struct knote *kn, long hint) 1548{ 1549 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; 1550 int ready; 1551 1552 BPFD_LOCK_ASSERT(d); 1553 ready = bpf_ready(d); 1554 if (ready) { 1555 kn->kn_data = d->bd_slen; 1556 if (d->bd_hbuf) 1557 kn->kn_data += d->bd_hlen; 1558 } 1559 else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1560 callout_reset(&d->bd_callout, d->bd_rtout, 1561 bpf_timed_out, d); 1562 d->bd_state = BPF_WAITING; 1563 } 1564 1565 return (ready); 1566} 1567 1568/* 1569 * Incoming linkage from device drivers. Process the packet pkt, of length 1570 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1571 * by each process' filter, and if accepted, stashed into the corresponding 1572 * buffer. 1573 */ 1574void 1575bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1576{ 1577 struct bpf_d *d; 1578 u_int slen; 1579 int gottime; 1580 struct timeval tv; 1581 1582 gottime = 0; 1583 BPFIF_LOCK(bp); 1584 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 1585 BPFD_LOCK(d); 1586 ++d->bd_rcount; 1587 /* 1588 * NB: We dont call BPF_CHECK_DIRECTION() here since there is no 1589 * way for the caller to indiciate to us whether this packet 1590 * is inbound or outbound. In the bpf_mtap() routines, we use 1591 * the interface pointers on the mbuf to figure it out. 1592 */ 1593#ifdef BPF_JITTER 1594 if (bpf_jitter_enable != 0 && d->bd_bfilter != NULL) 1595 slen = (*(d->bd_bfilter->func))(pkt, pktlen, pktlen); 1596 else 1597#endif 1598 slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen); 1599 if (slen != 0) { 1600 d->bd_fcount++; 1601 if (!gottime) { 1602 microtime(&tv); 1603 gottime = 1; 1604 } 1605#ifdef MAC 1606 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 1607#endif 1608 catchpacket(d, pkt, pktlen, slen, 1609 bpf_append_bytes, &tv); 1610 } 1611 BPFD_UNLOCK(d); 1612 } 1613 BPFIF_UNLOCK(bp); 1614} 1615 1616#define BPF_CHECK_DIRECTION(d, r, i) \ 1617 (((d)->bd_direction == BPF_D_IN && (r) != (i)) || \ 1618 ((d)->bd_direction == BPF_D_OUT && (r) == (i))) 1619 1620/* 1621 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1622 */ 1623void 1624bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1625{ 1626 struct bpf_d *d; 1627 u_int pktlen, slen; 1628 int gottime; 1629 struct timeval tv; 1630 1631 /* Skip outgoing duplicate packets. */ 1632 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1633 m->m_flags &= ~M_PROMISC; 1634 return; 1635 } 1636 1637 gottime = 0; 1638 1639 pktlen = m_length(m, NULL); 1640 1641 BPFIF_LOCK(bp); 1642 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 1643 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) 1644 continue; 1645 BPFD_LOCK(d); 1646 ++d->bd_rcount; 1647#ifdef BPF_JITTER 1648 /* XXX We cannot handle multiple mbufs. */ 1649 if (bpf_jitter_enable != 0 && d->bd_bfilter != NULL && 1650 m->m_next == NULL) 1651 slen = (*(d->bd_bfilter->func))(mtod(m, u_char *), 1652 pktlen, pktlen); 1653 else 1654#endif 1655 slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0); 1656 if (slen != 0) { 1657 d->bd_fcount++; 1658 if (!gottime) { 1659 microtime(&tv); 1660 gottime = 1; 1661 } 1662#ifdef MAC 1663 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 1664#endif 1665 catchpacket(d, (u_char *)m, pktlen, slen, 1666 bpf_append_mbuf, &tv); 1667 } 1668 BPFD_UNLOCK(d); 1669 } 1670 BPFIF_UNLOCK(bp); 1671} 1672 1673/* 1674 * Incoming linkage from device drivers, when packet is in 1675 * an mbuf chain and to be prepended by a contiguous header. 1676 */ 1677void 1678bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1679{ 1680 struct mbuf mb; 1681 struct bpf_d *d; 1682 u_int pktlen, slen; 1683 int gottime; 1684 struct timeval tv; 1685 1686 /* Skip outgoing duplicate packets. */ 1687 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1688 m->m_flags &= ~M_PROMISC; 1689 return; 1690 } 1691 1692 gottime = 0; 1693 1694 pktlen = m_length(m, NULL); 1695 /* 1696 * Craft on-stack mbuf suitable for passing to bpf_filter. 1697 * Note that we cut corners here; we only setup what's 1698 * absolutely needed--this mbuf should never go anywhere else. 1699 */ 1700 mb.m_next = m; 1701 mb.m_data = data; 1702 mb.m_len = dlen; 1703 pktlen += dlen; 1704 1705 BPFIF_LOCK(bp); 1706 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 1707 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) 1708 continue; 1709 BPFD_LOCK(d); 1710 ++d->bd_rcount; 1711 slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); 1712 if (slen != 0) { 1713 d->bd_fcount++; 1714 if (!gottime) { 1715 microtime(&tv); 1716 gottime = 1; 1717 } 1718#ifdef MAC 1719 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 1720#endif 1721 catchpacket(d, (u_char *)&mb, pktlen, slen, 1722 bpf_append_mbuf, &tv); 1723 } 1724 BPFD_UNLOCK(d); 1725 } 1726 BPFIF_UNLOCK(bp); 1727} 1728 1729#undef BPF_CHECK_DIRECTION 1730 1731/* 1732 * Move the packet data from interface memory (pkt) into the 1733 * store buffer. "cpfn" is the routine called to do the actual data 1734 * transfer. bcopy is passed in to copy contiguous chunks, while 1735 * bpf_append_mbuf is passed in to copy mbuf chains. In the latter case, 1736 * pkt is really an mbuf. 1737 */ 1738static void 1739catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1740 void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int), 1741 struct timeval *tv) 1742{ 1743 struct bpf_hdr hdr; 1744 int totlen, curlen; 1745 int hdrlen = d->bd_bif->bif_hdrlen; 1746 int do_wakeup = 0; 1747 1748 BPFD_LOCK_ASSERT(d); 1749 1750 /* 1751 * Detect whether user space has released a buffer back to us, and if 1752 * so, move it from being a hold buffer to a free buffer. This may 1753 * not be the best place to do it (for example, we might only want to 1754 * run this check if we need the space), but for now it's a reliable 1755 * spot to do it. 1756 */ 1757 if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) { 1758 d->bd_fbuf = d->bd_hbuf; 1759 d->bd_hbuf = NULL; 1760 d->bd_hlen = 0; 1761 bpf_buf_reclaimed(d); 1762 } 1763 1764 /* 1765 * Figure out how many bytes to move. If the packet is 1766 * greater or equal to the snapshot length, transfer that 1767 * much. Otherwise, transfer the whole packet (unless 1768 * we hit the buffer size limit). 1769 */ 1770 totlen = hdrlen + min(snaplen, pktlen); 1771 if (totlen > d->bd_bufsize) 1772 totlen = d->bd_bufsize; 1773 1774 /* 1775 * Round up the end of the previous packet to the next longword. 1776 * 1777 * Drop the packet if there's no room and no hope of room 1778 * If the packet would overflow the storage buffer or the storage 1779 * buffer is considered immutable by the buffer model, try to rotate 1780 * the buffer and wakeup pending processes. 1781 */ 1782 curlen = BPF_WORDALIGN(d->bd_slen); 1783 if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) { 1784 if (d->bd_fbuf == NULL) { 1785 /* 1786 * There's no room in the store buffer, and no 1787 * prospect of room, so drop the packet. Notify the 1788 * buffer model. 1789 */ 1790 bpf_buffull(d); 1791 ++d->bd_dcount; 1792 return; 1793 } 1794 ROTATE_BUFFERS(d); 1795 do_wakeup = 1; 1796 curlen = 0; 1797 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) 1798 /* 1799 * Immediate mode is set, or the read timeout has already 1800 * expired during a select call. A packet arrived, so the 1801 * reader should be woken up. 1802 */ 1803 do_wakeup = 1; 1804 1805 /* 1806 * Append the bpf header. Note we append the actual header size, but 1807 * move forward the length of the header plus padding. 1808 */ 1809 bzero(&hdr, sizeof(hdr)); 1810 hdr.bh_tstamp = *tv; 1811 hdr.bh_datalen = pktlen; 1812 hdr.bh_hdrlen = hdrlen; 1813 hdr.bh_caplen = totlen - hdrlen; 1814 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr)); 1815 1816 /* 1817 * Copy the packet data into the store buffer and update its length. 1818 */ 1819 (*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, hdr.bh_caplen); 1820 d->bd_slen = curlen + totlen; 1821 1822 if (do_wakeup) 1823 bpf_wakeup(d); 1824} 1825 1826/* 1827 * Free buffers currently in use by a descriptor. 1828 * Called on close. 1829 */ 1830static void 1831bpf_freed(struct bpf_d *d) 1832{ 1833 1834 /* 1835 * We don't need to lock out interrupts since this descriptor has 1836 * been detached from its interface and it yet hasn't been marked 1837 * free. 1838 */ 1839 bpf_free(d); 1840 if (d->bd_rfilter) { 1841 free((caddr_t)d->bd_rfilter, M_BPF); 1842#ifdef BPF_JITTER 1843 bpf_destroy_jit_filter(d->bd_bfilter); 1844#endif 1845 } 1846 if (d->bd_wfilter) 1847 free((caddr_t)d->bd_wfilter, M_BPF); 1848 mtx_destroy(&d->bd_mtx); 1849} 1850 1851/* 1852 * Attach an interface to bpf. dlt is the link layer type; hdrlen is the 1853 * fixed size of the link header (variable length headers not yet supported). 1854 */ 1855void 1856bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen) 1857{ 1858 1859 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); 1860} 1861 1862/* 1863 * Attach an interface to bpf. ifp is a pointer to the structure 1864 * defining the interface to be attached, dlt is the link layer type, 1865 * and hdrlen is the fixed size of the link header (variable length 1866 * headers are not yet supporrted). 1867 */ 1868void 1869bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 1870{ 1871 struct bpf_if *bp; 1872 1873 bp = malloc(sizeof(*bp), M_BPF, M_NOWAIT | M_ZERO); 1874 if (bp == NULL) 1875 panic("bpfattach"); 1876 1877 LIST_INIT(&bp->bif_dlist); 1878 bp->bif_ifp = ifp; 1879 bp->bif_dlt = dlt; 1880 mtx_init(&bp->bif_mtx, "bpf interface lock", NULL, MTX_DEF); 1881 KASSERT(*driverp == NULL, ("bpfattach2: driverp already initialized")); 1882 *driverp = bp; 1883 1884 mtx_lock(&bpf_mtx); 1885 LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next); 1886 mtx_unlock(&bpf_mtx); 1887 1888 /* 1889 * Compute the length of the bpf header. This is not necessarily 1890 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1891 * that the network layer header begins on a longword boundary (for 1892 * performance reasons and to alleviate alignment restrictions). 1893 */ 1894 bp->bif_hdrlen = BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen; 1895 1896 if (bootverbose) 1897 if_printf(ifp, "bpf attached\n"); 1898} 1899 1900/* 1901 * Detach bpf from an interface. This involves detaching each descriptor 1902 * associated with the interface, and leaving bd_bif NULL. Notify each 1903 * descriptor as it's detached so that any sleepers wake up and get 1904 * ENXIO. 1905 */ 1906void 1907bpfdetach(struct ifnet *ifp) 1908{ 1909 struct bpf_if *bp; 1910 struct bpf_d *d; 1911 1912 /* Locate BPF interface information */ 1913 mtx_lock(&bpf_mtx); 1914 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 1915 if (ifp == bp->bif_ifp) 1916 break; 1917 } 1918 1919 /* Interface wasn't attached */ 1920 if ((bp == NULL) || (bp->bif_ifp == NULL)) { 1921 mtx_unlock(&bpf_mtx); 1922 printf("bpfdetach: %s was not attached\n", ifp->if_xname); 1923 return; 1924 } 1925 1926 LIST_REMOVE(bp, bif_next); 1927 mtx_unlock(&bpf_mtx); 1928 1929 while ((d = LIST_FIRST(&bp->bif_dlist)) != NULL) { 1930 bpf_detachd(d); 1931 BPFD_LOCK(d); 1932 bpf_wakeup(d); 1933 BPFD_UNLOCK(d); 1934 } 1935 1936 mtx_destroy(&bp->bif_mtx); 1937 free(bp, M_BPF); 1938} 1939 1940/* 1941 * Get a list of available data link type of the interface. 1942 */ 1943static int 1944bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 1945{ 1946 int n, error; 1947 struct ifnet *ifp; 1948 struct bpf_if *bp; 1949 1950 ifp = d->bd_bif->bif_ifp; 1951 n = 0; 1952 error = 0; 1953 mtx_lock(&bpf_mtx); 1954 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 1955 if (bp->bif_ifp != ifp) 1956 continue; 1957 if (bfl->bfl_list != NULL) { 1958 if (n >= bfl->bfl_len) { 1959 mtx_unlock(&bpf_mtx); 1960 return (ENOMEM); 1961 } 1962 error = copyout(&bp->bif_dlt, 1963 bfl->bfl_list + n, sizeof(u_int)); 1964 } 1965 n++; 1966 } 1967 mtx_unlock(&bpf_mtx); 1968 bfl->bfl_len = n; 1969 return (error); 1970} 1971 1972/* 1973 * Set the data link type of a BPF instance. 1974 */ 1975static int 1976bpf_setdlt(struct bpf_d *d, u_int dlt) 1977{ 1978 int error, opromisc; 1979 struct ifnet *ifp; 1980 struct bpf_if *bp; 1981 1982 if (d->bd_bif->bif_dlt == dlt) 1983 return (0); 1984 ifp = d->bd_bif->bif_ifp; 1985 mtx_lock(&bpf_mtx); 1986 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 1987 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 1988 break; 1989 } 1990 mtx_unlock(&bpf_mtx); 1991 if (bp != NULL) { 1992 opromisc = d->bd_promisc; 1993 bpf_detachd(d); 1994 bpf_attachd(d, bp); 1995 BPFD_LOCK(d); 1996 reset_d(d); 1997 BPFD_UNLOCK(d); 1998 if (opromisc) { 1999 error = ifpromisc(bp->bif_ifp, 1); 2000 if (error) 2001 if_printf(bp->bif_ifp, 2002 "bpf_setdlt: ifpromisc failed (%d)\n", 2003 error); 2004 else 2005 d->bd_promisc = 1; 2006 } 2007 } 2008 return (bp == NULL ? EINVAL : 0); 2009} 2010 2011static void 2012bpf_drvinit(void *unused) 2013{ 2014 struct cdev *dev; 2015 2016 mtx_init(&bpf_mtx, "bpf global lock", NULL, MTX_DEF); 2017 LIST_INIT(&bpf_iflist); 2018 2019 dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf"); 2020 /* For compatibility */ 2021 make_dev_alias(dev, "bpf0"); 2022 2023} 2024 2025static void 2026bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) 2027{ 2028 2029 bzero(d, sizeof(*d)); 2030 BPFD_LOCK_ASSERT(bd); 2031 d->bd_structsize = sizeof(*d); 2032 d->bd_immediate = bd->bd_immediate; 2033 d->bd_promisc = bd->bd_promisc; 2034 d->bd_hdrcmplt = bd->bd_hdrcmplt; 2035 d->bd_direction = bd->bd_direction; 2036 d->bd_feedback = bd->bd_feedback; 2037 d->bd_async = bd->bd_async; 2038 d->bd_rcount = bd->bd_rcount; 2039 d->bd_dcount = bd->bd_dcount; 2040 d->bd_fcount = bd->bd_fcount; 2041 d->bd_sig = bd->bd_sig; 2042 d->bd_slen = bd->bd_slen; 2043 d->bd_hlen = bd->bd_hlen; 2044 d->bd_bufsize = bd->bd_bufsize; 2045 d->bd_pid = bd->bd_pid; 2046 strlcpy(d->bd_ifname, 2047 bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ); 2048 d->bd_locked = bd->bd_locked; 2049 d->bd_wcount = bd->bd_wcount; 2050 d->bd_wdcount = bd->bd_wdcount; 2051 d->bd_wfcount = bd->bd_wfcount; 2052 d->bd_zcopy = bd->bd_zcopy; 2053 d->bd_bufmode = bd->bd_bufmode; 2054} 2055 2056static int 2057bpf_stats_sysctl(SYSCTL_HANDLER_ARGS) 2058{ 2059 struct xbpf_d *xbdbuf, *xbd; 2060 int index, error; 2061 struct bpf_if *bp; 2062 struct bpf_d *bd; 2063 2064 /* 2065 * XXX This is not technically correct. It is possible for non 2066 * privileged users to open bpf devices. It would make sense 2067 * if the users who opened the devices were able to retrieve 2068 * the statistics for them, too. 2069 */ 2070 error = priv_check(req->td, PRIV_NET_BPF); 2071 if (error) 2072 return (error); 2073 if (req->oldptr == NULL) 2074 return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd))); 2075 if (bpf_bpfd_cnt == 0) 2076 return (SYSCTL_OUT(req, 0, 0)); 2077 xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK); 2078 mtx_lock(&bpf_mtx); 2079 if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) { 2080 mtx_unlock(&bpf_mtx); 2081 free(xbdbuf, M_BPF); 2082 return (ENOMEM); 2083 } 2084 index = 0; 2085 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2086 BPFIF_LOCK(bp); 2087 LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { 2088 xbd = &xbdbuf[index++]; 2089 BPFD_LOCK(bd); 2090 bpfstats_fill_xbpf(xbd, bd); 2091 BPFD_UNLOCK(bd); 2092 } 2093 BPFIF_UNLOCK(bp); 2094 } 2095 mtx_unlock(&bpf_mtx); 2096 error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd)); 2097 free(xbdbuf, M_BPF); 2098 return (error); 2099} 2100 2101SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL); 2102 2103#else /* !DEV_BPF && !NETGRAPH_BPF */ 2104/* 2105 * NOP stubs to allow bpf-using drivers to load and function. 2106 * 2107 * A 'better' implementation would allow the core bpf functionality 2108 * to be loaded at runtime. 2109 */ 2110static struct bpf_if bp_null; 2111 2112void 2113bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 2114{ 2115} 2116 2117void 2118bpf_mtap(struct bpf_if *bp, struct mbuf *m) 2119{ 2120} 2121 2122void 2123bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m) 2124{ 2125} 2126 2127void 2128bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2129{ 2130 2131 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); 2132} 2133 2134void 2135bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2136{ 2137 2138 *driverp = &bp_null; 2139} 2140 2141void 2142bpfdetach(struct ifnet *ifp) 2143{ 2144} 2145 2146u_int 2147bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) 2148{ 2149 return -1; /* "no filter" behaviour */ 2150} 2151 2152int 2153bpf_validate(const struct bpf_insn *f, int len) 2154{ 2155 return 0; /* false */ 2156} 2157 2158#endif /* !DEV_BPF && !NETGRAPH_BPF */ 2159