bpf.c revision 182462
1/*- 2 * Copyright (c) 1990, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from the Stanford/CMU enet packet filter, 6 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 7 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 8 * Berkeley Laboratory. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 35 */ 36 37#include <sys/cdefs.h> 38__FBSDID("$FreeBSD: head/sys/net/bpf.c 182462 2008-08-29 20:34:06Z jkim $"); 39 40#include "opt_bpf.h" 41#include "opt_mac.h" 42#include "opt_netgraph.h" 43 44#include <sys/types.h> 45#include <sys/param.h> 46#include <sys/systm.h> 47#include <sys/conf.h> 48#include <sys/fcntl.h> 49#include <sys/malloc.h> 50#include <sys/mbuf.h> 51#include <sys/time.h> 52#include <sys/priv.h> 53#include <sys/proc.h> 54#include <sys/signalvar.h> 55#include <sys/filio.h> 56#include <sys/sockio.h> 57#include <sys/ttycom.h> 58#include <sys/uio.h> 59 60#include <sys/event.h> 61#include <sys/file.h> 62#include <sys/poll.h> 63#include <sys/proc.h> 64 65#include <sys/socket.h> 66 67#include <net/if.h> 68#include <net/bpf.h> 69#include <net/bpf_buffer.h> 70#ifdef BPF_JITTER 71#include <net/bpf_jitter.h> 72#endif 73#include <net/bpf_zerocopy.h> 74#include <net/bpfdesc.h> 75 76#include <netinet/in.h> 77#include <netinet/if_ether.h> 78#include <sys/kernel.h> 79#include <sys/sysctl.h> 80 81#include <net80211/ieee80211_freebsd.h> 82 83#include <security/mac/mac_framework.h> 84 85MALLOC_DEFINE(M_BPF, "BPF", "BPF data"); 86 87#if defined(DEV_BPF) || defined(NETGRAPH_BPF) 88 89#define PRINET 26 /* interruptible */ 90 91/* 92 * bpf_iflist is a list of BPF interface structures, each corresponding to a 93 * specific DLT. The same network interface might have several BPF interface 94 * structures registered by different layers in the stack (i.e., 802.11 95 * frames, ethernet frames, etc). 96 */ 97static LIST_HEAD(, bpf_if) bpf_iflist; 98static struct mtx bpf_mtx; /* bpf global lock */ 99static int bpf_bpfd_cnt; 100 101static void bpf_attachd(struct bpf_d *, struct bpf_if *); 102static void bpf_detachd(struct bpf_d *); 103static void bpf_freed(struct bpf_d *); 104static int bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **, 105 struct sockaddr *, int *, struct bpf_insn *); 106static int bpf_setif(struct bpf_d *, struct ifreq *); 107static void bpf_timed_out(void *); 108static __inline void 109 bpf_wakeup(struct bpf_d *); 110static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 111 void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int), 112 struct timeval *); 113static void reset_d(struct bpf_d *); 114static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd); 115static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 116static int bpf_setdlt(struct bpf_d *, u_int); 117static void filt_bpfdetach(struct knote *); 118static int filt_bpfread(struct knote *, long); 119static void bpf_drvinit(void *); 120static int bpf_stats_sysctl(SYSCTL_HANDLER_ARGS); 121 122SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW, 0, "bpf sysctl"); 123int bpf_maxinsns = BPF_MAXINSNS; 124SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW, 125 &bpf_maxinsns, 0, "Maximum bpf program instructions"); 126static int bpf_zerocopy_enable = 0; 127SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW, 128 &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions"); 129SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_RW, 130 bpf_stats_sysctl, "bpf statistics portal"); 131 132static d_open_t bpfopen; 133static d_read_t bpfread; 134static d_write_t bpfwrite; 135static d_ioctl_t bpfioctl; 136static d_poll_t bpfpoll; 137static d_kqfilter_t bpfkqfilter; 138 139static struct cdevsw bpf_cdevsw = { 140 .d_version = D_VERSION, 141 .d_open = bpfopen, 142 .d_read = bpfread, 143 .d_write = bpfwrite, 144 .d_ioctl = bpfioctl, 145 .d_poll = bpfpoll, 146 .d_name = "bpf", 147 .d_kqfilter = bpfkqfilter, 148}; 149 150static struct filterops bpfread_filtops = 151 { 1, NULL, filt_bpfdetach, filt_bpfread }; 152 153/* 154 * Wrapper functions for various buffering methods. If the set of buffer 155 * modes expands, we will probably want to introduce a switch data structure 156 * similar to protosw, et. 157 */ 158static void 159bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, 160 u_int len) 161{ 162 163 BPFD_LOCK_ASSERT(d); 164 165 switch (d->bd_bufmode) { 166 case BPF_BUFMODE_BUFFER: 167 return (bpf_buffer_append_bytes(d, buf, offset, src, len)); 168 169 case BPF_BUFMODE_ZBUF: 170 d->bd_zcopy++; 171 return (bpf_zerocopy_append_bytes(d, buf, offset, src, len)); 172 173 default: 174 panic("bpf_buf_append_bytes"); 175 } 176} 177 178static void 179bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src, 180 u_int len) 181{ 182 183 BPFD_LOCK_ASSERT(d); 184 185 switch (d->bd_bufmode) { 186 case BPF_BUFMODE_BUFFER: 187 return (bpf_buffer_append_mbuf(d, buf, offset, src, len)); 188 189 case BPF_BUFMODE_ZBUF: 190 d->bd_zcopy++; 191 return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len)); 192 193 default: 194 panic("bpf_buf_append_mbuf"); 195 } 196} 197 198/* 199 * This function gets called when the free buffer is re-assigned. 200 */ 201static void 202bpf_buf_reclaimed(struct bpf_d *d) 203{ 204 205 BPFD_LOCK_ASSERT(d); 206 207 switch (d->bd_bufmode) { 208 case BPF_BUFMODE_BUFFER: 209 return; 210 211 case BPF_BUFMODE_ZBUF: 212 bpf_zerocopy_buf_reclaimed(d); 213 return; 214 215 default: 216 panic("bpf_buf_reclaimed"); 217 } 218} 219 220/* 221 * If the buffer mechanism has a way to decide that a held buffer can be made 222 * free, then it is exposed via the bpf_canfreebuf() interface. (1) is 223 * returned if the buffer can be discarded, (0) is returned if it cannot. 224 */ 225static int 226bpf_canfreebuf(struct bpf_d *d) 227{ 228 229 BPFD_LOCK_ASSERT(d); 230 231 switch (d->bd_bufmode) { 232 case BPF_BUFMODE_ZBUF: 233 return (bpf_zerocopy_canfreebuf(d)); 234 } 235 return (0); 236} 237 238/* 239 * Allow the buffer model to indicate that the current store buffer is 240 * immutable, regardless of the appearance of space. Return (1) if the 241 * buffer is writable, and (0) if not. 242 */ 243static int 244bpf_canwritebuf(struct bpf_d *d) 245{ 246 247 BPFD_LOCK_ASSERT(d); 248 249 switch (d->bd_bufmode) { 250 case BPF_BUFMODE_ZBUF: 251 return (bpf_zerocopy_canwritebuf(d)); 252 } 253 return (1); 254} 255 256/* 257 * Notify buffer model that an attempt to write to the store buffer has 258 * resulted in a dropped packet, in which case the buffer may be considered 259 * full. 260 */ 261static void 262bpf_buffull(struct bpf_d *d) 263{ 264 265 BPFD_LOCK_ASSERT(d); 266 267 switch (d->bd_bufmode) { 268 case BPF_BUFMODE_ZBUF: 269 bpf_zerocopy_buffull(d); 270 break; 271 } 272} 273 274/* 275 * Notify the buffer model that a buffer has moved into the hold position. 276 */ 277void 278bpf_bufheld(struct bpf_d *d) 279{ 280 281 BPFD_LOCK_ASSERT(d); 282 283 switch (d->bd_bufmode) { 284 case BPF_BUFMODE_ZBUF: 285 bpf_zerocopy_bufheld(d); 286 break; 287 } 288} 289 290static void 291bpf_free(struct bpf_d *d) 292{ 293 294 switch (d->bd_bufmode) { 295 case BPF_BUFMODE_BUFFER: 296 return (bpf_buffer_free(d)); 297 298 case BPF_BUFMODE_ZBUF: 299 return (bpf_zerocopy_free(d)); 300 301 default: 302 panic("bpf_buf_free"); 303 } 304} 305 306static int 307bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio) 308{ 309 310 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) 311 return (EOPNOTSUPP); 312 return (bpf_buffer_uiomove(d, buf, len, uio)); 313} 314 315static int 316bpf_ioctl_sblen(struct bpf_d *d, u_int *i) 317{ 318 319 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) 320 return (EOPNOTSUPP); 321 return (bpf_buffer_ioctl_sblen(d, i)); 322} 323 324static int 325bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i) 326{ 327 328 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 329 return (EOPNOTSUPP); 330 return (bpf_zerocopy_ioctl_getzmax(td, d, i)); 331} 332 333static int 334bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) 335{ 336 337 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 338 return (EOPNOTSUPP); 339 return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz)); 340} 341 342static int 343bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz) 344{ 345 346 if (d->bd_bufmode != BPF_BUFMODE_ZBUF) 347 return (EOPNOTSUPP); 348 return (bpf_zerocopy_ioctl_setzbuf(td, d, bz)); 349} 350 351/* 352 * General BPF functions. 353 */ 354static int 355bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp, 356 struct sockaddr *sockp, int *hdrlen, struct bpf_insn *wfilter) 357{ 358 const struct ieee80211_bpf_params *p; 359 struct ether_header *eh; 360 struct mbuf *m; 361 int error; 362 int len; 363 int hlen; 364 int slen; 365 366 /* 367 * Build a sockaddr based on the data link layer type. 368 * We do this at this level because the ethernet header 369 * is copied directly into the data field of the sockaddr. 370 * In the case of SLIP, there is no header and the packet 371 * is forwarded as is. 372 * Also, we are careful to leave room at the front of the mbuf 373 * for the link level header. 374 */ 375 switch (linktype) { 376 377 case DLT_SLIP: 378 sockp->sa_family = AF_INET; 379 hlen = 0; 380 break; 381 382 case DLT_EN10MB: 383 sockp->sa_family = AF_UNSPEC; 384 /* XXX Would MAXLINKHDR be better? */ 385 hlen = ETHER_HDR_LEN; 386 break; 387 388 case DLT_FDDI: 389 sockp->sa_family = AF_IMPLINK; 390 hlen = 0; 391 break; 392 393 case DLT_RAW: 394 sockp->sa_family = AF_UNSPEC; 395 hlen = 0; 396 break; 397 398 case DLT_NULL: 399 /* 400 * null interface types require a 4 byte pseudo header which 401 * corresponds to the address family of the packet. 402 */ 403 sockp->sa_family = AF_UNSPEC; 404 hlen = 4; 405 break; 406 407 case DLT_ATM_RFC1483: 408 /* 409 * en atm driver requires 4-byte atm pseudo header. 410 * though it isn't standard, vpi:vci needs to be 411 * specified anyway. 412 */ 413 sockp->sa_family = AF_UNSPEC; 414 hlen = 12; /* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */ 415 break; 416 417 case DLT_PPP: 418 sockp->sa_family = AF_UNSPEC; 419 hlen = 4; /* This should match PPP_HDRLEN */ 420 break; 421 422 case DLT_IEEE802_11: /* IEEE 802.11 wireless */ 423 sockp->sa_family = AF_IEEE80211; 424 hlen = 0; 425 break; 426 427 case DLT_IEEE802_11_RADIO: /* IEEE 802.11 wireless w/ phy params */ 428 sockp->sa_family = AF_IEEE80211; 429 sockp->sa_len = 12; /* XXX != 0 */ 430 hlen = sizeof(struct ieee80211_bpf_params); 431 break; 432 433 default: 434 return (EIO); 435 } 436 437 len = uio->uio_resid; 438 439 if (len - hlen > ifp->if_mtu) 440 return (EMSGSIZE); 441 442 if ((unsigned)len > MJUM16BYTES) 443 return (EIO); 444 445 if (len <= MHLEN) 446 MGETHDR(m, M_WAIT, MT_DATA); 447 else if (len <= MCLBYTES) 448 m = m_getcl(M_WAIT, MT_DATA, M_PKTHDR); 449 else 450 m = m_getjcl(M_WAIT, MT_DATA, M_PKTHDR, 451#if (MJUMPAGESIZE > MCLBYTES) 452 len <= MJUMPAGESIZE ? MJUMPAGESIZE : 453#endif 454 (len <= MJUM9BYTES ? MJUM9BYTES : MJUM16BYTES)); 455 m->m_pkthdr.len = m->m_len = len; 456 m->m_pkthdr.rcvif = NULL; 457 *mp = m; 458 459 if (m->m_len < hlen) { 460 error = EPERM; 461 goto bad; 462 } 463 464 error = uiomove(mtod(m, u_char *), len, uio); 465 if (error) 466 goto bad; 467 468 slen = bpf_filter(wfilter, mtod(m, u_char *), len, len); 469 if (slen == 0) { 470 error = EPERM; 471 goto bad; 472 } 473 474 /* Check for multicast destination */ 475 switch (linktype) { 476 case DLT_EN10MB: 477 eh = mtod(m, struct ether_header *); 478 if (ETHER_IS_MULTICAST(eh->ether_dhost)) { 479 if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost, 480 ETHER_ADDR_LEN) == 0) 481 m->m_flags |= M_BCAST; 482 else 483 m->m_flags |= M_MCAST; 484 } 485 break; 486 } 487 488 /* 489 * Make room for link header, and copy it to sockaddr 490 */ 491 if (hlen != 0) { 492 if (sockp->sa_family == AF_IEEE80211) { 493 /* 494 * Collect true length from the parameter header 495 * NB: sockp is known to be zero'd so if we do a 496 * short copy unspecified parameters will be 497 * zero. 498 * NB: packet may not be aligned after stripping 499 * bpf params 500 * XXX check ibp_vers 501 */ 502 p = mtod(m, const struct ieee80211_bpf_params *); 503 hlen = p->ibp_len; 504 if (hlen > sizeof(sockp->sa_data)) { 505 error = EINVAL; 506 goto bad; 507 } 508 } 509 bcopy(m->m_data, sockp->sa_data, hlen); 510 } 511 *hdrlen = hlen; 512 513 return (0); 514bad: 515 m_freem(m); 516 return (error); 517} 518 519/* 520 * Attach file to the bpf interface, i.e. make d listen on bp. 521 */ 522static void 523bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 524{ 525 /* 526 * Point d at bp, and add d to the interface's list of listeners. 527 * Finally, point the driver's bpf cookie at the interface so 528 * it will divert packets to bpf. 529 */ 530 BPFIF_LOCK(bp); 531 d->bd_bif = bp; 532 LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next); 533 534 bpf_bpfd_cnt++; 535 BPFIF_UNLOCK(bp); 536} 537 538/* 539 * Detach a file from its interface. 540 */ 541static void 542bpf_detachd(struct bpf_d *d) 543{ 544 int error; 545 struct bpf_if *bp; 546 struct ifnet *ifp; 547 548 bp = d->bd_bif; 549 BPFIF_LOCK(bp); 550 BPFD_LOCK(d); 551 ifp = d->bd_bif->bif_ifp; 552 553 /* 554 * Remove d from the interface's descriptor list. 555 */ 556 LIST_REMOVE(d, bd_next); 557 558 bpf_bpfd_cnt--; 559 d->bd_bif = NULL; 560 BPFD_UNLOCK(d); 561 BPFIF_UNLOCK(bp); 562 563 /* 564 * Check if this descriptor had requested promiscuous mode. 565 * If so, turn it off. 566 */ 567 if (d->bd_promisc) { 568 d->bd_promisc = 0; 569 error = ifpromisc(ifp, 0); 570 if (error != 0 && error != ENXIO) { 571 /* 572 * ENXIO can happen if a pccard is unplugged 573 * Something is really wrong if we were able to put 574 * the driver into promiscuous mode, but can't 575 * take it out. 576 */ 577 if_printf(bp->bif_ifp, 578 "bpf_detach: ifpromisc failed (%d)\n", error); 579 } 580 } 581} 582 583/* 584 * Close the descriptor by detaching it from its interface, 585 * deallocating its buffers, and marking it free. 586 */ 587static void 588bpf_dtor(void *data) 589{ 590 struct bpf_d *d = data; 591 592 BPFD_LOCK(d); 593 if (d->bd_state == BPF_WAITING) 594 callout_stop(&d->bd_callout); 595 d->bd_state = BPF_IDLE; 596 BPFD_UNLOCK(d); 597 funsetown(&d->bd_sigio); 598 mtx_lock(&bpf_mtx); 599 if (d->bd_bif) 600 bpf_detachd(d); 601 mtx_unlock(&bpf_mtx); 602 selwakeuppri(&d->bd_sel, PRINET); 603#ifdef MAC 604 mac_bpfdesc_destroy(d); 605#endif /* MAC */ 606 knlist_destroy(&d->bd_sel.si_note); 607 bpf_freed(d); 608 free(d, M_BPF); 609} 610 611/* 612 * Open ethernet device. Returns ENXIO for illegal minor device number, 613 * EBUSY if file is open by another process. 614 */ 615/* ARGSUSED */ 616static int 617bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td) 618{ 619 struct bpf_d *d; 620 int error; 621 622 MALLOC(d, struct bpf_d *, sizeof(*d), M_BPF, M_WAITOK | M_ZERO); 623 error = devfs_set_cdevpriv(d, bpf_dtor); 624 if (error != 0) { 625 free(d, M_BPF); 626 return (error); 627 } 628 629 /* 630 * For historical reasons, perform a one-time initialization call to 631 * the buffer routines, even though we're not yet committed to a 632 * particular buffer method. 633 */ 634 bpf_buffer_init(d); 635 d->bd_bufmode = BPF_BUFMODE_BUFFER; 636 d->bd_sig = SIGIO; 637 d->bd_direction = BPF_D_INOUT; 638 d->bd_pid = td->td_proc->p_pid; 639#ifdef MAC 640 mac_bpfdesc_init(d); 641 mac_bpfdesc_create(td->td_ucred, d); 642#endif 643 mtx_init(&d->bd_mtx, devtoname(dev), "bpf cdev lock", MTX_DEF); 644 callout_init(&d->bd_callout, CALLOUT_MPSAFE); 645 knlist_init(&d->bd_sel.si_note, &d->bd_mtx, NULL, NULL, NULL); 646 647 return (0); 648} 649 650/* 651 * bpfread - read next chunk of packets from buffers 652 */ 653static int 654bpfread(struct cdev *dev, struct uio *uio, int ioflag) 655{ 656 struct bpf_d *d; 657 int timed_out; 658 int error; 659 660 error = devfs_get_cdevpriv((void **)&d); 661 if (error != 0) 662 return (error); 663 664 /* 665 * Restrict application to use a buffer the same size as 666 * as kernel buffers. 667 */ 668 if (uio->uio_resid != d->bd_bufsize) 669 return (EINVAL); 670 671 BPFD_LOCK(d); 672 d->bd_pid = curthread->td_proc->p_pid; 673 if (d->bd_bufmode != BPF_BUFMODE_BUFFER) { 674 BPFD_UNLOCK(d); 675 return (EOPNOTSUPP); 676 } 677 if (d->bd_state == BPF_WAITING) 678 callout_stop(&d->bd_callout); 679 timed_out = (d->bd_state == BPF_TIMED_OUT); 680 d->bd_state = BPF_IDLE; 681 /* 682 * If the hold buffer is empty, then do a timed sleep, which 683 * ends when the timeout expires or when enough packets 684 * have arrived to fill the store buffer. 685 */ 686 while (d->bd_hbuf == NULL) { 687 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 688 /* 689 * A packet(s) either arrived since the previous 690 * read or arrived while we were asleep. 691 * Rotate the buffers and return what's here. 692 */ 693 ROTATE_BUFFERS(d); 694 break; 695 } 696 697 /* 698 * No data is available, check to see if the bpf device 699 * is still pointed at a real interface. If not, return 700 * ENXIO so that the userland process knows to rebind 701 * it before using it again. 702 */ 703 if (d->bd_bif == NULL) { 704 BPFD_UNLOCK(d); 705 return (ENXIO); 706 } 707 708 if (ioflag & O_NONBLOCK) { 709 BPFD_UNLOCK(d); 710 return (EWOULDBLOCK); 711 } 712 error = msleep(d, &d->bd_mtx, PRINET|PCATCH, 713 "bpf", d->bd_rtout); 714 if (error == EINTR || error == ERESTART) { 715 BPFD_UNLOCK(d); 716 return (error); 717 } 718 if (error == EWOULDBLOCK) { 719 /* 720 * On a timeout, return what's in the buffer, 721 * which may be nothing. If there is something 722 * in the store buffer, we can rotate the buffers. 723 */ 724 if (d->bd_hbuf) 725 /* 726 * We filled up the buffer in between 727 * getting the timeout and arriving 728 * here, so we don't need to rotate. 729 */ 730 break; 731 732 if (d->bd_slen == 0) { 733 BPFD_UNLOCK(d); 734 return (0); 735 } 736 ROTATE_BUFFERS(d); 737 break; 738 } 739 } 740 /* 741 * At this point, we know we have something in the hold slot. 742 */ 743 BPFD_UNLOCK(d); 744 745 /* 746 * Move data from hold buffer into user space. 747 * We know the entire buffer is transferred since 748 * we checked above that the read buffer is bpf_bufsize bytes. 749 * 750 * XXXRW: More synchronization needed here: what if a second thread 751 * issues a read on the same fd at the same time? Don't want this 752 * getting invalidated. 753 */ 754 error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio); 755 756 BPFD_LOCK(d); 757 d->bd_fbuf = d->bd_hbuf; 758 d->bd_hbuf = NULL; 759 d->bd_hlen = 0; 760 bpf_buf_reclaimed(d); 761 BPFD_UNLOCK(d); 762 763 return (error); 764} 765 766/* 767 * If there are processes sleeping on this descriptor, wake them up. 768 */ 769static __inline void 770bpf_wakeup(struct bpf_d *d) 771{ 772 773 BPFD_LOCK_ASSERT(d); 774 if (d->bd_state == BPF_WAITING) { 775 callout_stop(&d->bd_callout); 776 d->bd_state = BPF_IDLE; 777 } 778 wakeup(d); 779 if (d->bd_async && d->bd_sig && d->bd_sigio) 780 pgsigio(&d->bd_sigio, d->bd_sig, 0); 781 782 selwakeuppri(&d->bd_sel, PRINET); 783 KNOTE_LOCKED(&d->bd_sel.si_note, 0); 784} 785 786static void 787bpf_timed_out(void *arg) 788{ 789 struct bpf_d *d = (struct bpf_d *)arg; 790 791 BPFD_LOCK(d); 792 if (d->bd_state == BPF_WAITING) { 793 d->bd_state = BPF_TIMED_OUT; 794 if (d->bd_slen != 0) 795 bpf_wakeup(d); 796 } 797 BPFD_UNLOCK(d); 798} 799 800static int 801bpf_ready(struct bpf_d *d) 802{ 803 804 BPFD_LOCK_ASSERT(d); 805 806 if (!bpf_canfreebuf(d) && d->bd_hlen != 0) 807 return (1); 808 if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 809 d->bd_slen != 0) 810 return (1); 811 return (0); 812} 813 814static int 815bpfwrite(struct cdev *dev, struct uio *uio, int ioflag) 816{ 817 struct bpf_d *d; 818 struct ifnet *ifp; 819 struct mbuf *m, *mc; 820 struct sockaddr dst; 821 int error, hlen; 822 823 error = devfs_get_cdevpriv((void **)&d); 824 if (error != 0) 825 return (error); 826 827 d->bd_pid = curthread->td_proc->p_pid; 828 d->bd_wcount++; 829 if (d->bd_bif == NULL) { 830 d->bd_wdcount++; 831 return (ENXIO); 832 } 833 834 ifp = d->bd_bif->bif_ifp; 835 836 if ((ifp->if_flags & IFF_UP) == 0) { 837 d->bd_wdcount++; 838 return (ENETDOWN); 839 } 840 841 if (uio->uio_resid == 0) { 842 d->bd_wdcount++; 843 return (0); 844 } 845 846 bzero(&dst, sizeof(dst)); 847 m = NULL; 848 hlen = 0; 849 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp, 850 &m, &dst, &hlen, d->bd_wfilter); 851 if (error) { 852 d->bd_wdcount++; 853 return (error); 854 } 855 d->bd_wfcount++; 856 if (d->bd_hdrcmplt) 857 dst.sa_family = pseudo_AF_HDRCMPLT; 858 859 if (d->bd_feedback) { 860 mc = m_dup(m, M_DONTWAIT); 861 if (mc != NULL) 862 mc->m_pkthdr.rcvif = ifp; 863 /* Set M_PROMISC for outgoing packets to be discarded. */ 864 if (d->bd_direction == BPF_D_INOUT) 865 m->m_flags |= M_PROMISC; 866 } else 867 mc = NULL; 868 869 m->m_pkthdr.len -= hlen; 870 m->m_len -= hlen; 871 m->m_data += hlen; /* XXX */ 872 873#ifdef MAC 874 BPFD_LOCK(d); 875 mac_bpfdesc_create_mbuf(d, m); 876 if (mc != NULL) 877 mac_bpfdesc_create_mbuf(d, mc); 878 BPFD_UNLOCK(d); 879#endif 880 881 error = (*ifp->if_output)(ifp, m, &dst, NULL); 882 if (error) 883 d->bd_wdcount++; 884 885 if (mc != NULL) { 886 if (error == 0) 887 (*ifp->if_input)(ifp, mc); 888 else 889 m_freem(mc); 890 } 891 892 return (error); 893} 894 895/* 896 * Reset a descriptor by flushing its packet buffer and clearing the 897 * receive and drop counts. 898 */ 899static void 900reset_d(struct bpf_d *d) 901{ 902 903 mtx_assert(&d->bd_mtx, MA_OWNED); 904 if (d->bd_hbuf) { 905 /* Free the hold buffer. */ 906 d->bd_fbuf = d->bd_hbuf; 907 d->bd_hbuf = NULL; 908 bpf_buf_reclaimed(d); 909 } 910 d->bd_slen = 0; 911 d->bd_hlen = 0; 912 d->bd_rcount = 0; 913 d->bd_dcount = 0; 914 d->bd_fcount = 0; 915 d->bd_wcount = 0; 916 d->bd_wfcount = 0; 917 d->bd_wdcount = 0; 918 d->bd_zcopy = 0; 919} 920 921/* 922 * FIONREAD Check for read packet available. 923 * SIOCGIFADDR Get interface address - convenient hook to driver. 924 * BIOCGBLEN Get buffer len [for read()]. 925 * BIOCSETF Set read filter. 926 * BIOCSETFNR Set read filter without resetting descriptor. 927 * BIOCSETWF Set write filter. 928 * BIOCFLUSH Flush read packet buffer. 929 * BIOCPROMISC Put interface into promiscuous mode. 930 * BIOCGDLT Get link layer type. 931 * BIOCGETIF Get interface name. 932 * BIOCSETIF Set interface. 933 * BIOCSRTIMEOUT Set read timeout. 934 * BIOCGRTIMEOUT Get read timeout. 935 * BIOCGSTATS Get packet stats. 936 * BIOCIMMEDIATE Set immediate mode. 937 * BIOCVERSION Get filter language version. 938 * BIOCGHDRCMPLT Get "header already complete" flag 939 * BIOCSHDRCMPLT Set "header already complete" flag 940 * BIOCGDIRECTION Get packet direction flag 941 * BIOCSDIRECTION Set packet direction flag 942 * BIOCLOCK Set "locked" flag 943 * BIOCFEEDBACK Set packet feedback mode. 944 * BIOCSETZBUF Set current zero-copy buffer locations. 945 * BIOCGETZMAX Get maximum zero-copy buffer size. 946 * BIOCROTZBUF Force rotation of zero-copy buffer 947 * BIOCSETBUFMODE Set buffer mode. 948 * BIOCGETBUFMODE Get current buffer mode. 949 */ 950/* ARGSUSED */ 951static int 952bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, 953 struct thread *td) 954{ 955 struct bpf_d *d; 956 int error; 957 958 error = devfs_get_cdevpriv((void **)&d); 959 if (error != 0) 960 return (error); 961 962 /* 963 * Refresh PID associated with this descriptor. 964 */ 965 BPFD_LOCK(d); 966 d->bd_pid = td->td_proc->p_pid; 967 if (d->bd_state == BPF_WAITING) 968 callout_stop(&d->bd_callout); 969 d->bd_state = BPF_IDLE; 970 BPFD_UNLOCK(d); 971 972 if (d->bd_locked == 1) { 973 switch (cmd) { 974 case BIOCGBLEN: 975 case BIOCFLUSH: 976 case BIOCGDLT: 977 case BIOCGDLTLIST: 978 case BIOCGETIF: 979 case BIOCGRTIMEOUT: 980 case BIOCGSTATS: 981 case BIOCVERSION: 982 case BIOCGRSIG: 983 case BIOCGHDRCMPLT: 984 case BIOCFEEDBACK: 985 case FIONREAD: 986 case BIOCLOCK: 987 case BIOCSRTIMEOUT: 988 case BIOCIMMEDIATE: 989 case TIOCGPGRP: 990 case BIOCROTZBUF: 991 break; 992 default: 993 return (EPERM); 994 } 995 } 996 switch (cmd) { 997 998 default: 999 error = EINVAL; 1000 break; 1001 1002 /* 1003 * Check for read packet available. 1004 */ 1005 case FIONREAD: 1006 { 1007 int n; 1008 1009 BPFD_LOCK(d); 1010 n = d->bd_slen; 1011 if (d->bd_hbuf) 1012 n += d->bd_hlen; 1013 BPFD_UNLOCK(d); 1014 1015 *(int *)addr = n; 1016 break; 1017 } 1018 1019 case SIOCGIFADDR: 1020 { 1021 struct ifnet *ifp; 1022 1023 if (d->bd_bif == NULL) 1024 error = EINVAL; 1025 else { 1026 ifp = d->bd_bif->bif_ifp; 1027 error = (*ifp->if_ioctl)(ifp, cmd, addr); 1028 } 1029 break; 1030 } 1031 1032 /* 1033 * Get buffer len [for read()]. 1034 */ 1035 case BIOCGBLEN: 1036 *(u_int *)addr = d->bd_bufsize; 1037 break; 1038 1039 /* 1040 * Set buffer length. 1041 */ 1042 case BIOCSBLEN: 1043 error = bpf_ioctl_sblen(d, (u_int *)addr); 1044 break; 1045 1046 /* 1047 * Set link layer read filter. 1048 */ 1049 case BIOCSETF: 1050 case BIOCSETFNR: 1051 case BIOCSETWF: 1052 error = bpf_setf(d, (struct bpf_program *)addr, cmd); 1053 break; 1054 1055 /* 1056 * Flush read packet buffer. 1057 */ 1058 case BIOCFLUSH: 1059 BPFD_LOCK(d); 1060 reset_d(d); 1061 BPFD_UNLOCK(d); 1062 break; 1063 1064 /* 1065 * Put interface into promiscuous mode. 1066 */ 1067 case BIOCPROMISC: 1068 if (d->bd_bif == NULL) { 1069 /* 1070 * No interface attached yet. 1071 */ 1072 error = EINVAL; 1073 break; 1074 } 1075 if (d->bd_promisc == 0) { 1076 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1077 if (error == 0) 1078 d->bd_promisc = 1; 1079 } 1080 break; 1081 1082 /* 1083 * Get current data link type. 1084 */ 1085 case BIOCGDLT: 1086 if (d->bd_bif == NULL) 1087 error = EINVAL; 1088 else 1089 *(u_int *)addr = d->bd_bif->bif_dlt; 1090 break; 1091 1092 /* 1093 * Get a list of supported data link types. 1094 */ 1095 case BIOCGDLTLIST: 1096 if (d->bd_bif == NULL) 1097 error = EINVAL; 1098 else 1099 error = bpf_getdltlist(d, (struct bpf_dltlist *)addr); 1100 break; 1101 1102 /* 1103 * Set data link type. 1104 */ 1105 case BIOCSDLT: 1106 if (d->bd_bif == NULL) 1107 error = EINVAL; 1108 else 1109 error = bpf_setdlt(d, *(u_int *)addr); 1110 break; 1111 1112 /* 1113 * Get interface name. 1114 */ 1115 case BIOCGETIF: 1116 if (d->bd_bif == NULL) 1117 error = EINVAL; 1118 else { 1119 struct ifnet *const ifp = d->bd_bif->bif_ifp; 1120 struct ifreq *const ifr = (struct ifreq *)addr; 1121 1122 strlcpy(ifr->ifr_name, ifp->if_xname, 1123 sizeof(ifr->ifr_name)); 1124 } 1125 break; 1126 1127 /* 1128 * Set interface. 1129 */ 1130 case BIOCSETIF: 1131 error = bpf_setif(d, (struct ifreq *)addr); 1132 break; 1133 1134 /* 1135 * Set read timeout. 1136 */ 1137 case BIOCSRTIMEOUT: 1138 { 1139 struct timeval *tv = (struct timeval *)addr; 1140 1141 /* 1142 * Subtract 1 tick from tvtohz() since this isn't 1143 * a one-shot timer. 1144 */ 1145 if ((error = itimerfix(tv)) == 0) 1146 d->bd_rtout = tvtohz(tv) - 1; 1147 break; 1148 } 1149 1150 /* 1151 * Get read timeout. 1152 */ 1153 case BIOCGRTIMEOUT: 1154 { 1155 struct timeval *tv = (struct timeval *)addr; 1156 1157 tv->tv_sec = d->bd_rtout / hz; 1158 tv->tv_usec = (d->bd_rtout % hz) * tick; 1159 break; 1160 } 1161 1162 /* 1163 * Get packet stats. 1164 */ 1165 case BIOCGSTATS: 1166 { 1167 struct bpf_stat *bs = (struct bpf_stat *)addr; 1168 1169 /* XXXCSJP overflow */ 1170 bs->bs_recv = d->bd_rcount; 1171 bs->bs_drop = d->bd_dcount; 1172 break; 1173 } 1174 1175 /* 1176 * Set immediate mode. 1177 */ 1178 case BIOCIMMEDIATE: 1179 d->bd_immediate = *(u_int *)addr; 1180 break; 1181 1182 case BIOCVERSION: 1183 { 1184 struct bpf_version *bv = (struct bpf_version *)addr; 1185 1186 bv->bv_major = BPF_MAJOR_VERSION; 1187 bv->bv_minor = BPF_MINOR_VERSION; 1188 break; 1189 } 1190 1191 /* 1192 * Get "header already complete" flag 1193 */ 1194 case BIOCGHDRCMPLT: 1195 *(u_int *)addr = d->bd_hdrcmplt; 1196 break; 1197 1198 /* 1199 * Set "header already complete" flag 1200 */ 1201 case BIOCSHDRCMPLT: 1202 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1203 break; 1204 1205 /* 1206 * Get packet direction flag 1207 */ 1208 case BIOCGDIRECTION: 1209 *(u_int *)addr = d->bd_direction; 1210 break; 1211 1212 /* 1213 * Set packet direction flag 1214 */ 1215 case BIOCSDIRECTION: 1216 { 1217 u_int direction; 1218 1219 direction = *(u_int *)addr; 1220 switch (direction) { 1221 case BPF_D_IN: 1222 case BPF_D_INOUT: 1223 case BPF_D_OUT: 1224 d->bd_direction = direction; 1225 break; 1226 default: 1227 error = EINVAL; 1228 } 1229 } 1230 break; 1231 1232 case BIOCFEEDBACK: 1233 d->bd_feedback = *(u_int *)addr; 1234 break; 1235 1236 case BIOCLOCK: 1237 d->bd_locked = 1; 1238 break; 1239 1240 case FIONBIO: /* Non-blocking I/O */ 1241 break; 1242 1243 case FIOASYNC: /* Send signal on receive packets */ 1244 d->bd_async = *(int *)addr; 1245 break; 1246 1247 case FIOSETOWN: 1248 error = fsetown(*(int *)addr, &d->bd_sigio); 1249 break; 1250 1251 case FIOGETOWN: 1252 *(int *)addr = fgetown(&d->bd_sigio); 1253 break; 1254 1255 /* This is deprecated, FIOSETOWN should be used instead. */ 1256 case TIOCSPGRP: 1257 error = fsetown(-(*(int *)addr), &d->bd_sigio); 1258 break; 1259 1260 /* This is deprecated, FIOGETOWN should be used instead. */ 1261 case TIOCGPGRP: 1262 *(int *)addr = -fgetown(&d->bd_sigio); 1263 break; 1264 1265 case BIOCSRSIG: /* Set receive signal */ 1266 { 1267 u_int sig; 1268 1269 sig = *(u_int *)addr; 1270 1271 if (sig >= NSIG) 1272 error = EINVAL; 1273 else 1274 d->bd_sig = sig; 1275 break; 1276 } 1277 case BIOCGRSIG: 1278 *(u_int *)addr = d->bd_sig; 1279 break; 1280 1281 case BIOCGETBUFMODE: 1282 *(u_int *)addr = d->bd_bufmode; 1283 break; 1284 1285 case BIOCSETBUFMODE: 1286 /* 1287 * Allow the buffering mode to be changed as long as we 1288 * haven't yet committed to a particular mode. Our 1289 * definition of commitment, for now, is whether or not a 1290 * buffer has been allocated or an interface attached, since 1291 * that's the point where things get tricky. 1292 */ 1293 switch (*(u_int *)addr) { 1294 case BPF_BUFMODE_BUFFER: 1295 break; 1296 1297 case BPF_BUFMODE_ZBUF: 1298 if (bpf_zerocopy_enable) 1299 break; 1300 /* FALLSTHROUGH */ 1301 1302 default: 1303 return (EINVAL); 1304 } 1305 1306 BPFD_LOCK(d); 1307 if (d->bd_sbuf != NULL || d->bd_hbuf != NULL || 1308 d->bd_fbuf != NULL || d->bd_bif != NULL) { 1309 BPFD_UNLOCK(d); 1310 return (EBUSY); 1311 } 1312 d->bd_bufmode = *(u_int *)addr; 1313 BPFD_UNLOCK(d); 1314 break; 1315 1316 case BIOCGETZMAX: 1317 return (bpf_ioctl_getzmax(td, d, (size_t *)addr)); 1318 1319 case BIOCSETZBUF: 1320 return (bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr)); 1321 1322 case BIOCROTZBUF: 1323 return (bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr)); 1324 } 1325 return (error); 1326} 1327 1328/* 1329 * Set d's packet filter program to fp. If this file already has a filter, 1330 * free it and replace it. Returns EINVAL for bogus requests. 1331 */ 1332static int 1333bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) 1334{ 1335 struct bpf_insn *fcode, *old; 1336 u_int wfilter, flen, size; 1337#ifdef BPF_JITTER 1338 bpf_jit_filter *ofunc; 1339#endif 1340 1341 if (cmd == BIOCSETWF) { 1342 old = d->bd_wfilter; 1343 wfilter = 1; 1344#ifdef BPF_JITTER 1345 ofunc = NULL; 1346#endif 1347 } else { 1348 wfilter = 0; 1349 old = d->bd_rfilter; 1350#ifdef BPF_JITTER 1351 ofunc = d->bd_bfilter; 1352#endif 1353 } 1354 if (fp->bf_insns == NULL) { 1355 if (fp->bf_len != 0) 1356 return (EINVAL); 1357 BPFD_LOCK(d); 1358 if (wfilter) 1359 d->bd_wfilter = NULL; 1360 else { 1361 d->bd_rfilter = NULL; 1362#ifdef BPF_JITTER 1363 d->bd_bfilter = NULL; 1364#endif 1365 if (cmd == BIOCSETF) 1366 reset_d(d); 1367 } 1368 BPFD_UNLOCK(d); 1369 if (old != NULL) 1370 free((caddr_t)old, M_BPF); 1371#ifdef BPF_JITTER 1372 if (ofunc != NULL) 1373 bpf_destroy_jit_filter(ofunc); 1374#endif 1375 return (0); 1376 } 1377 flen = fp->bf_len; 1378 if (flen > bpf_maxinsns) 1379 return (EINVAL); 1380 1381 size = flen * sizeof(*fp->bf_insns); 1382 fcode = (struct bpf_insn *)malloc(size, M_BPF, M_WAITOK); 1383 if (copyin((caddr_t)fp->bf_insns, (caddr_t)fcode, size) == 0 && 1384 bpf_validate(fcode, (int)flen)) { 1385 BPFD_LOCK(d); 1386 if (wfilter) 1387 d->bd_wfilter = fcode; 1388 else { 1389 d->bd_rfilter = fcode; 1390#ifdef BPF_JITTER 1391 d->bd_bfilter = bpf_jitter(fcode, flen); 1392#endif 1393 if (cmd == BIOCSETF) 1394 reset_d(d); 1395 } 1396 BPFD_UNLOCK(d); 1397 if (old != NULL) 1398 free((caddr_t)old, M_BPF); 1399#ifdef BPF_JITTER 1400 if (ofunc != NULL) 1401 bpf_destroy_jit_filter(ofunc); 1402#endif 1403 1404 return (0); 1405 } 1406 free((caddr_t)fcode, M_BPF); 1407 return (EINVAL); 1408} 1409 1410/* 1411 * Detach a file from its current interface (if attached at all) and attach 1412 * to the interface indicated by the name stored in ifr. 1413 * Return an errno or 0. 1414 */ 1415static int 1416bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1417{ 1418 struct bpf_if *bp; 1419 struct ifnet *theywant; 1420 1421 theywant = ifunit(ifr->ifr_name); 1422 if (theywant == NULL || theywant->if_bpf == NULL) 1423 return (ENXIO); 1424 1425 bp = theywant->if_bpf; 1426 1427 /* 1428 * Behavior here depends on the buffering model. If we're using 1429 * kernel memory buffers, then we can allocate them here. If we're 1430 * using zero-copy, then the user process must have registered 1431 * buffers by the time we get here. If not, return an error. 1432 * 1433 * XXXRW: There are locking issues here with multi-threaded use: what 1434 * if two threads try to set the interface at once? 1435 */ 1436 switch (d->bd_bufmode) { 1437 case BPF_BUFMODE_BUFFER: 1438 if (d->bd_sbuf == NULL) 1439 bpf_buffer_alloc(d); 1440 KASSERT(d->bd_sbuf != NULL, ("bpf_setif: bd_sbuf NULL")); 1441 break; 1442 1443 case BPF_BUFMODE_ZBUF: 1444 if (d->bd_sbuf == NULL) 1445 return (EINVAL); 1446 break; 1447 1448 default: 1449 panic("bpf_setif: bufmode %d", d->bd_bufmode); 1450 } 1451 if (bp != d->bd_bif) { 1452 if (d->bd_bif) 1453 /* 1454 * Detach if attached to something else. 1455 */ 1456 bpf_detachd(d); 1457 1458 bpf_attachd(d, bp); 1459 } 1460 BPFD_LOCK(d); 1461 reset_d(d); 1462 BPFD_UNLOCK(d); 1463 return (0); 1464} 1465 1466/* 1467 * Support for select() and poll() system calls 1468 * 1469 * Return true iff the specific operation will not block indefinitely. 1470 * Otherwise, return false but make a note that a selwakeup() must be done. 1471 */ 1472static int 1473bpfpoll(struct cdev *dev, int events, struct thread *td) 1474{ 1475 struct bpf_d *d; 1476 int revents; 1477 1478 if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL) 1479 return (events & 1480 (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM)); 1481 1482 /* 1483 * Refresh PID associated with this descriptor. 1484 */ 1485 revents = events & (POLLOUT | POLLWRNORM); 1486 BPFD_LOCK(d); 1487 d->bd_pid = td->td_proc->p_pid; 1488 if (events & (POLLIN | POLLRDNORM)) { 1489 if (bpf_ready(d)) 1490 revents |= events & (POLLIN | POLLRDNORM); 1491 else { 1492 selrecord(td, &d->bd_sel); 1493 /* Start the read timeout if necessary. */ 1494 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1495 callout_reset(&d->bd_callout, d->bd_rtout, 1496 bpf_timed_out, d); 1497 d->bd_state = BPF_WAITING; 1498 } 1499 } 1500 } 1501 BPFD_UNLOCK(d); 1502 return (revents); 1503} 1504 1505/* 1506 * Support for kevent() system call. Register EVFILT_READ filters and 1507 * reject all others. 1508 */ 1509int 1510bpfkqfilter(struct cdev *dev, struct knote *kn) 1511{ 1512 struct bpf_d *d; 1513 1514 if (devfs_get_cdevpriv((void **)&d) != 0 || 1515 kn->kn_filter != EVFILT_READ) 1516 return (1); 1517 1518 /* 1519 * Refresh PID associated with this descriptor. 1520 */ 1521 BPFD_LOCK(d); 1522 d->bd_pid = curthread->td_proc->p_pid; 1523 kn->kn_fop = &bpfread_filtops; 1524 kn->kn_hook = d; 1525 knlist_add(&d->bd_sel.si_note, kn, 1); 1526 BPFD_UNLOCK(d); 1527 1528 return (0); 1529} 1530 1531static void 1532filt_bpfdetach(struct knote *kn) 1533{ 1534 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; 1535 1536 knlist_remove(&d->bd_sel.si_note, kn, 0); 1537} 1538 1539static int 1540filt_bpfread(struct knote *kn, long hint) 1541{ 1542 struct bpf_d *d = (struct bpf_d *)kn->kn_hook; 1543 int ready; 1544 1545 BPFD_LOCK_ASSERT(d); 1546 ready = bpf_ready(d); 1547 if (ready) { 1548 kn->kn_data = d->bd_slen; 1549 if (d->bd_hbuf) 1550 kn->kn_data += d->bd_hlen; 1551 } 1552 else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1553 callout_reset(&d->bd_callout, d->bd_rtout, 1554 bpf_timed_out, d); 1555 d->bd_state = BPF_WAITING; 1556 } 1557 1558 return (ready); 1559} 1560 1561/* 1562 * Incoming linkage from device drivers. Process the packet pkt, of length 1563 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1564 * by each process' filter, and if accepted, stashed into the corresponding 1565 * buffer. 1566 */ 1567void 1568bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1569{ 1570 struct bpf_d *d; 1571 u_int slen; 1572 int gottime; 1573 struct timeval tv; 1574 1575 gottime = 0; 1576 BPFIF_LOCK(bp); 1577 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 1578 BPFD_LOCK(d); 1579 ++d->bd_rcount; 1580 /* 1581 * NB: We dont call BPF_CHECK_DIRECTION() here since there is no 1582 * way for the caller to indiciate to us whether this packet 1583 * is inbound or outbound. In the bpf_mtap() routines, we use 1584 * the interface pointers on the mbuf to figure it out. 1585 */ 1586#ifdef BPF_JITTER 1587 if (bpf_jitter_enable != 0 && d->bd_bfilter != NULL) 1588 slen = (*(d->bd_bfilter->func))(pkt, pktlen, pktlen); 1589 else 1590#endif 1591 slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen); 1592 if (slen != 0) { 1593 d->bd_fcount++; 1594 if (!gottime) { 1595 microtime(&tv); 1596 gottime = 1; 1597 } 1598#ifdef MAC 1599 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 1600#endif 1601 catchpacket(d, pkt, pktlen, slen, 1602 bpf_append_bytes, &tv); 1603 } 1604 BPFD_UNLOCK(d); 1605 } 1606 BPFIF_UNLOCK(bp); 1607} 1608 1609#define BPF_CHECK_DIRECTION(d, r, i) \ 1610 (((d)->bd_direction == BPF_D_IN && (r) != (i)) || \ 1611 ((d)->bd_direction == BPF_D_OUT && (r) == (i))) 1612 1613/* 1614 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1615 */ 1616void 1617bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1618{ 1619 struct bpf_d *d; 1620 u_int pktlen, slen; 1621 int gottime; 1622 struct timeval tv; 1623 1624 /* Skip outgoing duplicate packets. */ 1625 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1626 m->m_flags &= ~M_PROMISC; 1627 return; 1628 } 1629 1630 gottime = 0; 1631 1632 pktlen = m_length(m, NULL); 1633 1634 BPFIF_LOCK(bp); 1635 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 1636 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) 1637 continue; 1638 BPFD_LOCK(d); 1639 ++d->bd_rcount; 1640#ifdef BPF_JITTER 1641 /* XXX We cannot handle multiple mbufs. */ 1642 if (bpf_jitter_enable != 0 && d->bd_bfilter != NULL && 1643 m->m_next == NULL) 1644 slen = (*(d->bd_bfilter->func))(mtod(m, u_char *), 1645 pktlen, pktlen); 1646 else 1647#endif 1648 slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0); 1649 if (slen != 0) { 1650 d->bd_fcount++; 1651 if (!gottime) { 1652 microtime(&tv); 1653 gottime = 1; 1654 } 1655#ifdef MAC 1656 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 1657#endif 1658 catchpacket(d, (u_char *)m, pktlen, slen, 1659 bpf_append_mbuf, &tv); 1660 } 1661 BPFD_UNLOCK(d); 1662 } 1663 BPFIF_UNLOCK(bp); 1664} 1665 1666/* 1667 * Incoming linkage from device drivers, when packet is in 1668 * an mbuf chain and to be prepended by a contiguous header. 1669 */ 1670void 1671bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1672{ 1673 struct mbuf mb; 1674 struct bpf_d *d; 1675 u_int pktlen, slen; 1676 int gottime; 1677 struct timeval tv; 1678 1679 /* Skip outgoing duplicate packets. */ 1680 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1681 m->m_flags &= ~M_PROMISC; 1682 return; 1683 } 1684 1685 gottime = 0; 1686 1687 pktlen = m_length(m, NULL); 1688 /* 1689 * Craft on-stack mbuf suitable for passing to bpf_filter. 1690 * Note that we cut corners here; we only setup what's 1691 * absolutely needed--this mbuf should never go anywhere else. 1692 */ 1693 mb.m_next = m; 1694 mb.m_data = data; 1695 mb.m_len = dlen; 1696 pktlen += dlen; 1697 1698 BPFIF_LOCK(bp); 1699 LIST_FOREACH(d, &bp->bif_dlist, bd_next) { 1700 if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp)) 1701 continue; 1702 BPFD_LOCK(d); 1703 ++d->bd_rcount; 1704 slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0); 1705 if (slen != 0) { 1706 d->bd_fcount++; 1707 if (!gottime) { 1708 microtime(&tv); 1709 gottime = 1; 1710 } 1711#ifdef MAC 1712 if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0) 1713#endif 1714 catchpacket(d, (u_char *)&mb, pktlen, slen, 1715 bpf_append_mbuf, &tv); 1716 } 1717 BPFD_UNLOCK(d); 1718 } 1719 BPFIF_UNLOCK(bp); 1720} 1721 1722#undef BPF_CHECK_DIRECTION 1723 1724/* 1725 * Move the packet data from interface memory (pkt) into the 1726 * store buffer. "cpfn" is the routine called to do the actual data 1727 * transfer. bcopy is passed in to copy contiguous chunks, while 1728 * bpf_append_mbuf is passed in to copy mbuf chains. In the latter case, 1729 * pkt is really an mbuf. 1730 */ 1731static void 1732catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1733 void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int), 1734 struct timeval *tv) 1735{ 1736 struct bpf_hdr hdr; 1737 int totlen, curlen; 1738 int hdrlen = d->bd_bif->bif_hdrlen; 1739 int do_wakeup = 0; 1740 1741 BPFD_LOCK_ASSERT(d); 1742 1743 /* 1744 * Detect whether user space has released a buffer back to us, and if 1745 * so, move it from being a hold buffer to a free buffer. This may 1746 * not be the best place to do it (for example, we might only want to 1747 * run this check if we need the space), but for now it's a reliable 1748 * spot to do it. 1749 */ 1750 if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) { 1751 d->bd_fbuf = d->bd_hbuf; 1752 d->bd_hbuf = NULL; 1753 d->bd_hlen = 0; 1754 bpf_buf_reclaimed(d); 1755 } 1756 1757 /* 1758 * Figure out how many bytes to move. If the packet is 1759 * greater or equal to the snapshot length, transfer that 1760 * much. Otherwise, transfer the whole packet (unless 1761 * we hit the buffer size limit). 1762 */ 1763 totlen = hdrlen + min(snaplen, pktlen); 1764 if (totlen > d->bd_bufsize) 1765 totlen = d->bd_bufsize; 1766 1767 /* 1768 * Round up the end of the previous packet to the next longword. 1769 * 1770 * Drop the packet if there's no room and no hope of room 1771 * If the packet would overflow the storage buffer or the storage 1772 * buffer is considered immutable by the buffer model, try to rotate 1773 * the buffer and wakeup pending processes. 1774 */ 1775 curlen = BPF_WORDALIGN(d->bd_slen); 1776 if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) { 1777 if (d->bd_fbuf == NULL) { 1778 /* 1779 * There's no room in the store buffer, and no 1780 * prospect of room, so drop the packet. Notify the 1781 * buffer model. 1782 */ 1783 bpf_buffull(d); 1784 ++d->bd_dcount; 1785 return; 1786 } 1787 ROTATE_BUFFERS(d); 1788 do_wakeup = 1; 1789 curlen = 0; 1790 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) 1791 /* 1792 * Immediate mode is set, or the read timeout has already 1793 * expired during a select call. A packet arrived, so the 1794 * reader should be woken up. 1795 */ 1796 do_wakeup = 1; 1797 1798 /* 1799 * Append the bpf header. Note we append the actual header size, but 1800 * move forward the length of the header plus padding. 1801 */ 1802 bzero(&hdr, sizeof(hdr)); 1803 hdr.bh_tstamp = *tv; 1804 hdr.bh_datalen = pktlen; 1805 hdr.bh_hdrlen = hdrlen; 1806 hdr.bh_caplen = totlen - hdrlen; 1807 bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr)); 1808 1809 /* 1810 * Copy the packet data into the store buffer and update its length. 1811 */ 1812 (*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, hdr.bh_caplen); 1813 d->bd_slen = curlen + totlen; 1814 1815 if (do_wakeup) 1816 bpf_wakeup(d); 1817} 1818 1819/* 1820 * Free buffers currently in use by a descriptor. 1821 * Called on close. 1822 */ 1823static void 1824bpf_freed(struct bpf_d *d) 1825{ 1826 1827 /* 1828 * We don't need to lock out interrupts since this descriptor has 1829 * been detached from its interface and it yet hasn't been marked 1830 * free. 1831 */ 1832 bpf_free(d); 1833 if (d->bd_rfilter) { 1834 free((caddr_t)d->bd_rfilter, M_BPF); 1835#ifdef BPF_JITTER 1836 bpf_destroy_jit_filter(d->bd_bfilter); 1837#endif 1838 } 1839 if (d->bd_wfilter) 1840 free((caddr_t)d->bd_wfilter, M_BPF); 1841 mtx_destroy(&d->bd_mtx); 1842} 1843 1844/* 1845 * Attach an interface to bpf. dlt is the link layer type; hdrlen is the 1846 * fixed size of the link header (variable length headers not yet supported). 1847 */ 1848void 1849bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen) 1850{ 1851 1852 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); 1853} 1854 1855/* 1856 * Attach an interface to bpf. ifp is a pointer to the structure 1857 * defining the interface to be attached, dlt is the link layer type, 1858 * and hdrlen is the fixed size of the link header (variable length 1859 * headers are not yet supporrted). 1860 */ 1861void 1862bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 1863{ 1864 struct bpf_if *bp; 1865 1866 bp = malloc(sizeof(*bp), M_BPF, M_NOWAIT | M_ZERO); 1867 if (bp == NULL) 1868 panic("bpfattach"); 1869 1870 LIST_INIT(&bp->bif_dlist); 1871 bp->bif_ifp = ifp; 1872 bp->bif_dlt = dlt; 1873 mtx_init(&bp->bif_mtx, "bpf interface lock", NULL, MTX_DEF); 1874 KASSERT(*driverp == NULL, ("bpfattach2: driverp already initialized")); 1875 *driverp = bp; 1876 1877 mtx_lock(&bpf_mtx); 1878 LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next); 1879 mtx_unlock(&bpf_mtx); 1880 1881 /* 1882 * Compute the length of the bpf header. This is not necessarily 1883 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1884 * that the network layer header begins on a longword boundary (for 1885 * performance reasons and to alleviate alignment restrictions). 1886 */ 1887 bp->bif_hdrlen = BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen; 1888 1889 if (bootverbose) 1890 if_printf(ifp, "bpf attached\n"); 1891} 1892 1893/* 1894 * Detach bpf from an interface. This involves detaching each descriptor 1895 * associated with the interface, and leaving bd_bif NULL. Notify each 1896 * descriptor as it's detached so that any sleepers wake up and get 1897 * ENXIO. 1898 */ 1899void 1900bpfdetach(struct ifnet *ifp) 1901{ 1902 struct bpf_if *bp; 1903 struct bpf_d *d; 1904 1905 /* Locate BPF interface information */ 1906 mtx_lock(&bpf_mtx); 1907 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 1908 if (ifp == bp->bif_ifp) 1909 break; 1910 } 1911 1912 /* Interface wasn't attached */ 1913 if ((bp == NULL) || (bp->bif_ifp == NULL)) { 1914 mtx_unlock(&bpf_mtx); 1915 printf("bpfdetach: %s was not attached\n", ifp->if_xname); 1916 return; 1917 } 1918 1919 LIST_REMOVE(bp, bif_next); 1920 mtx_unlock(&bpf_mtx); 1921 1922 while ((d = LIST_FIRST(&bp->bif_dlist)) != NULL) { 1923 bpf_detachd(d); 1924 BPFD_LOCK(d); 1925 bpf_wakeup(d); 1926 BPFD_UNLOCK(d); 1927 } 1928 1929 mtx_destroy(&bp->bif_mtx); 1930 free(bp, M_BPF); 1931} 1932 1933/* 1934 * Get a list of available data link type of the interface. 1935 */ 1936static int 1937bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 1938{ 1939 int n, error; 1940 struct ifnet *ifp; 1941 struct bpf_if *bp; 1942 1943 ifp = d->bd_bif->bif_ifp; 1944 n = 0; 1945 error = 0; 1946 mtx_lock(&bpf_mtx); 1947 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 1948 if (bp->bif_ifp != ifp) 1949 continue; 1950 if (bfl->bfl_list != NULL) { 1951 if (n >= bfl->bfl_len) { 1952 mtx_unlock(&bpf_mtx); 1953 return (ENOMEM); 1954 } 1955 error = copyout(&bp->bif_dlt, 1956 bfl->bfl_list + n, sizeof(u_int)); 1957 } 1958 n++; 1959 } 1960 mtx_unlock(&bpf_mtx); 1961 bfl->bfl_len = n; 1962 return (error); 1963} 1964 1965/* 1966 * Set the data link type of a BPF instance. 1967 */ 1968static int 1969bpf_setdlt(struct bpf_d *d, u_int dlt) 1970{ 1971 int error, opromisc; 1972 struct ifnet *ifp; 1973 struct bpf_if *bp; 1974 1975 if (d->bd_bif->bif_dlt == dlt) 1976 return (0); 1977 ifp = d->bd_bif->bif_ifp; 1978 mtx_lock(&bpf_mtx); 1979 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 1980 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 1981 break; 1982 } 1983 mtx_unlock(&bpf_mtx); 1984 if (bp != NULL) { 1985 opromisc = d->bd_promisc; 1986 bpf_detachd(d); 1987 bpf_attachd(d, bp); 1988 BPFD_LOCK(d); 1989 reset_d(d); 1990 BPFD_UNLOCK(d); 1991 if (opromisc) { 1992 error = ifpromisc(bp->bif_ifp, 1); 1993 if (error) 1994 if_printf(bp->bif_ifp, 1995 "bpf_setdlt: ifpromisc failed (%d)\n", 1996 error); 1997 else 1998 d->bd_promisc = 1; 1999 } 2000 } 2001 return (bp == NULL ? EINVAL : 0); 2002} 2003 2004static void 2005bpf_drvinit(void *unused) 2006{ 2007 struct cdev *dev; 2008 2009 mtx_init(&bpf_mtx, "bpf global lock", NULL, MTX_DEF); 2010 LIST_INIT(&bpf_iflist); 2011 2012 dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf"); 2013 /* For compatibility */ 2014 make_dev_alias(dev, "bpf0"); 2015 2016} 2017 2018static void 2019bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd) 2020{ 2021 2022 bzero(d, sizeof(*d)); 2023 BPFD_LOCK_ASSERT(bd); 2024 d->bd_structsize = sizeof(*d); 2025 d->bd_immediate = bd->bd_immediate; 2026 d->bd_promisc = bd->bd_promisc; 2027 d->bd_hdrcmplt = bd->bd_hdrcmplt; 2028 d->bd_direction = bd->bd_direction; 2029 d->bd_feedback = bd->bd_feedback; 2030 d->bd_async = bd->bd_async; 2031 d->bd_rcount = bd->bd_rcount; 2032 d->bd_dcount = bd->bd_dcount; 2033 d->bd_fcount = bd->bd_fcount; 2034 d->bd_sig = bd->bd_sig; 2035 d->bd_slen = bd->bd_slen; 2036 d->bd_hlen = bd->bd_hlen; 2037 d->bd_bufsize = bd->bd_bufsize; 2038 d->bd_pid = bd->bd_pid; 2039 strlcpy(d->bd_ifname, 2040 bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ); 2041 d->bd_locked = bd->bd_locked; 2042 d->bd_wcount = bd->bd_wcount; 2043 d->bd_wdcount = bd->bd_wdcount; 2044 d->bd_wfcount = bd->bd_wfcount; 2045 d->bd_zcopy = bd->bd_zcopy; 2046 d->bd_bufmode = bd->bd_bufmode; 2047} 2048 2049static int 2050bpf_stats_sysctl(SYSCTL_HANDLER_ARGS) 2051{ 2052 struct xbpf_d *xbdbuf, *xbd; 2053 int index, error; 2054 struct bpf_if *bp; 2055 struct bpf_d *bd; 2056 2057 /* 2058 * XXX This is not technically correct. It is possible for non 2059 * privileged users to open bpf devices. It would make sense 2060 * if the users who opened the devices were able to retrieve 2061 * the statistics for them, too. 2062 */ 2063 error = priv_check(req->td, PRIV_NET_BPF); 2064 if (error) 2065 return (error); 2066 if (req->oldptr == NULL) 2067 return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd))); 2068 if (bpf_bpfd_cnt == 0) 2069 return (SYSCTL_OUT(req, 0, 0)); 2070 xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK); 2071 mtx_lock(&bpf_mtx); 2072 if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) { 2073 mtx_unlock(&bpf_mtx); 2074 free(xbdbuf, M_BPF); 2075 return (ENOMEM); 2076 } 2077 index = 0; 2078 LIST_FOREACH(bp, &bpf_iflist, bif_next) { 2079 BPFIF_LOCK(bp); 2080 LIST_FOREACH(bd, &bp->bif_dlist, bd_next) { 2081 xbd = &xbdbuf[index++]; 2082 BPFD_LOCK(bd); 2083 bpfstats_fill_xbpf(xbd, bd); 2084 BPFD_UNLOCK(bd); 2085 } 2086 BPFIF_UNLOCK(bp); 2087 } 2088 mtx_unlock(&bpf_mtx); 2089 error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd)); 2090 free(xbdbuf, M_BPF); 2091 return (error); 2092} 2093 2094SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL); 2095 2096#else /* !DEV_BPF && !NETGRAPH_BPF */ 2097/* 2098 * NOP stubs to allow bpf-using drivers to load and function. 2099 * 2100 * A 'better' implementation would allow the core bpf functionality 2101 * to be loaded at runtime. 2102 */ 2103static struct bpf_if bp_null; 2104 2105void 2106bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 2107{ 2108} 2109 2110void 2111bpf_mtap(struct bpf_if *bp, struct mbuf *m) 2112{ 2113} 2114 2115void 2116bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m) 2117{ 2118} 2119 2120void 2121bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2122{ 2123 2124 bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf); 2125} 2126 2127void 2128bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2129{ 2130 2131 *driverp = &bp_null; 2132} 2133 2134void 2135bpfdetach(struct ifnet *ifp) 2136{ 2137} 2138 2139u_int 2140bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen) 2141{ 2142 return -1; /* "no filter" behaviour */ 2143} 2144 2145int 2146bpf_validate(const struct bpf_insn *f, int len) 2147{ 2148 return 0; /* false */ 2149} 2150 2151#endif /* !DEV_BPF && !NETGRAPH_BPF */ 2152