bpf.c revision 227309
134192Sjdp/*-
255687Sjdp * Copyright (c) 1990, 1991, 1993
334192Sjdp *	The Regents of the University of California.  All rights reserved.
434192Sjdp *
534192Sjdp * This code is derived from the Stanford/CMU enet packet filter,
634192Sjdp * (net/enet.c) distributed as part of 4.3BSD, and code contributed
734192Sjdp * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
834192Sjdp * Berkeley Laboratory.
934192Sjdp *
1034192Sjdp * Redistribution and use in source and binary forms, with or without
1134192Sjdp * modification, are permitted provided that the following conditions
1234192Sjdp * are met:
1334192Sjdp * 1. Redistributions of source code must retain the above copyright
1434192Sjdp *    notice, this list of conditions and the following disclaimer.
1534192Sjdp * 2. Redistributions in binary form must reproduce the above copyright
1634192Sjdp *    notice, this list of conditions and the following disclaimer in the
1734192Sjdp *    documentation and/or other materials provided with the distribution.
1834192Sjdp * 4. Neither the name of the University nor the names of its contributors
1934192Sjdp *    may be used to endorse or promote products derived from this software
2034192Sjdp *    without specific prior written permission.
2134192Sjdp *
2234192Sjdp * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
2334192Sjdp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2434192Sjdp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2550476Speter * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
2634192Sjdp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
2734192Sjdp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
2834192Sjdp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2934192Sjdp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
3034192Sjdp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
3176224Sobrien * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
3234192Sjdp * SUCH DAMAGE.
3350608Sjdp *
3434192Sjdp *      @(#)bpf.c	8.4 (Berkeley) 1/9/95
3576224Sobrien */
3635529Sdfr
3734192Sjdp#include <sys/cdefs.h>
3834192Sjdp__FBSDID("$FreeBSD: head/sys/net/bpf.c 227309 2011-11-07 15:43:11Z ed $");
39115396Skan
4045501Sjdp#include "opt_bpf.h"
4145501Sjdp#include "opt_compat.h"
42127250Speter#include "opt_netgraph.h"
43127250Speter
44127250Speter#include <sys/types.h>
45127250Speter#include <sys/param.h>
46127250Speter#include <sys/systm.h>
47127250Speter#include <sys/conf.h>
48127250Speter#include <sys/fcntl.h>
49127250Speter#include <sys/jail.h>
50127250Speter#include <sys/malloc.h>
5134192Sjdp#include <sys/mbuf.h>
52119013Sgordon#include <sys/time.h>
5334192Sjdp#include <sys/priv.h>
54127250Speter#include <sys/proc.h>
55127250Speter#include <sys/signalvar.h>
56127250Speter#include <sys/filio.h>
5734192Sjdp#include <sys/sockio.h>
5834192Sjdp#include <sys/ttycom.h>
5934192Sjdp#include <sys/uio.h>
6034192Sjdp
6134192Sjdp#include <sys/event.h>
6234192Sjdp#include <sys/file.h>
6334192Sjdp#include <sys/poll.h>
6434192Sjdp#include <sys/proc.h>
6534192Sjdp
66133063Sdfr#include <sys/socket.h>
67133063Sdfr
68133063Sdfr#include <net/if.h>
69133063Sdfr#include <net/bpf.h>
70133063Sdfr#include <net/bpf_buffer.h>
71133063Sdfr#ifdef BPF_JITTER
7250609Sjdp#include <net/bpf_jitter.h>
7334192Sjdp#endif
7434192Sjdp#include <net/bpf_zerocopy.h>
7555687Sjdp#include <net/bpfdesc.h>
7650608Sjdp#include <net/vnet.h>
7760938Sjake
7850608Sjdp#include <netinet/in.h>
7950608Sjdp#include <netinet/if_ether.h>
8050608Sjdp#include <sys/kernel.h>
8160938Sjake#include <sys/sysctl.h>
8250608Sjdp
8363870Sjdp#include <net80211/ieee80211_freebsd.h>
8455687Sjdp
8555687Sjdp#include <security/mac/mac_framework.h>
8655687Sjdp
8734192SjdpMALLOC_DEFINE(M_BPF, "BPF", "BPF data");
8834192Sjdp
8934192Sjdp#if defined(DEV_BPF) || defined(NETGRAPH_BPF)
9034192Sjdp
9134192Sjdp#define PRINET  26			/* interruptible */
9234192Sjdp
9362801Sjdp#define	SIZEOF_BPF_HDR(type)	\
9462801Sjdp    (offsetof(type, bh_hdrlen) + sizeof(((type *)0)->bh_hdrlen))
9562801Sjdp
9662801Sjdp#ifdef COMPAT_FREEBSD32
9762801Sjdp#include <sys/mount.h>
9862801Sjdp#include <compat/freebsd32/freebsd32.h>
9962801Sjdp#define BPF_ALIGNMENT32 sizeof(int32_t)
10062801Sjdp#define BPF_WORDALIGN32(x) (((x)+(BPF_ALIGNMENT32-1))&~(BPF_ALIGNMENT32-1))
10162801Sjdp
10262801Sjdp#ifndef BURN_BRIDGES
10362801Sjdp/*
10462801Sjdp * 32-bit version of structure prepended to each packet.  We use this header
10562801Sjdp * instead of the standard one for 32-bit streams.  We mark the a stream as
10662801Sjdp * 32-bit the first time we see a 32-bit compat ioctl request.
10762801Sjdp */
10862801Sjdpstruct bpf_hdr32 {
10962801Sjdp	struct timeval32 bh_tstamp;	/* time stamp */
11034192Sjdp	uint32_t	bh_caplen;	/* length of captured portion */
11134192Sjdp	uint32_t	bh_datalen;	/* original length of packet */
11234192Sjdp	uint16_t	bh_hdrlen;	/* length of bpf header (this struct
11334192Sjdp					   plus alignment padding) */
11434192Sjdp};
11550977Sjdp#endif
11650977Sjdp
11750977Sjdpstruct bpf_program32 {
11850977Sjdp	u_int bf_len;
11934192Sjdp	uint32_t bf_insns;
12034192Sjdp};
12134192Sjdp
12234192Sjdpstruct bpf_dltlist32 {
12334192Sjdp	u_int	bfl_len;
12434192Sjdp	u_int	bfl_list;
12538467Sjb};
12638467Sjb
12734192Sjdp#define	BIOCSETF32	_IOW('B', 103, struct bpf_program32)
12834192Sjdp#define	BIOCSRTIMEOUT32	_IOW('B', 109, struct timeval32)
12934192Sjdp#define	BIOCGRTIMEOUT32	_IOR('B', 110, struct timeval32)
130116511Smdodd#define	BIOCGDLTLIST32	_IOWR('B', 121, struct bpf_dltlist32)
13134192Sjdp#define	BIOCSETWF32	_IOW('B', 123, struct bpf_program32)
13234192Sjdp#define	BIOCSETFNR32	_IOW('B', 130, struct bpf_program32)
13334192Sjdp#endif
13434192Sjdp
13534192Sjdp/*
13634192Sjdp * bpf_iflist is a list of BPF interface structures, each corresponding to a
13734192Sjdp * specific DLT.  The same network interface might have several BPF interface
13838467Sjb * structures registered by different layers in the stack (i.e., 802.11
13934192Sjdp * frames, ethernet frames, etc).
14038467Sjb */
14134192Sjdpstatic LIST_HEAD(, bpf_if)	bpf_iflist;
14238467Sjbstatic struct mtx	bpf_mtx;		/* bpf global lock */
14334192Sjdpstatic int		bpf_bpfd_cnt;
14450610Sjdp
14534192Sjdpstatic void	bpf_attachd(struct bpf_d *, struct bpf_if *);
146133063Sdfrstatic void	bpf_detachd(struct bpf_d *);
147133063Sdfrstatic void	bpf_freed(struct bpf_d *);
148133063Sdfrstatic int	bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **,
149133063Sdfr		    struct sockaddr *, int *, struct bpf_insn *);
150133063Sdfrstatic int	bpf_setif(struct bpf_d *, struct ifreq *);
151133063Sdfrstatic void	bpf_timed_out(void *);
152133063Sdfrstatic __inline void
153133063Sdfr		bpf_wakeup(struct bpf_d *);
15434192Sjdpstatic void	catchpacket(struct bpf_d *, u_char *, u_int, u_int,
15545501Sjdp		    void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int),
15638467Sjb		    struct bintime *);
15734192Sjdpstatic void	reset_d(struct bpf_d *);
15838816Sdfrstatic int	 bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd);
15938816Sdfrstatic int	bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
16038467Sjbstatic int	bpf_setdlt(struct bpf_d *, u_int);
16134192Sjdpstatic void	filt_bpfdetach(struct knote *);
16238816Sdfrstatic int	filt_bpfread(struct knote *, long);
16338816Sdfrstatic void	bpf_drvinit(void *);
16438467Sjbstatic int	bpf_stats_sysctl(SYSCTL_HANDLER_ARGS);
16534192Sjdp
16634192SjdpSYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW, 0, "bpf sysctl");
16734192Sjdpint bpf_maxinsns = BPF_MAXINSNS;
16885004SdfrSYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW,
16934192Sjdp    &bpf_maxinsns, 0, "Maximum bpf program instructions");
17085004Sdfrstatic int bpf_zerocopy_enable = 0;
17134192SjdpSYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW,
17234192Sjdp    &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions");
17334192Sjdpstatic SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_MPSAFE | CTLFLAG_RW,
17434192Sjdp    bpf_stats_sysctl, "bpf statistics portal");
17534192Sjdp
17685677Speterstatic	d_open_t	bpfopen;
17785677Speterstatic	d_read_t	bpfread;
17834192Sjdpstatic	d_write_t	bpfwrite;
17934192Sjdpstatic	d_ioctl_t	bpfioctl;
18034192Sjdpstatic	d_poll_t	bpfpoll;
18134192Sjdpstatic	d_kqfilter_t	bpfkqfilter;
18234192Sjdp
183116511Smdoddstatic struct cdevsw bpf_cdevsw = {
18438740Sjdp	.d_version =	D_VERSION,
18556780Sjdp	.d_open =	bpfopen,
18663870Sjdp	.d_read =	bpfread,
187133063Sdfr	.d_write =	bpfwrite,
18835529Sdfr	.d_ioctl =	bpfioctl,
189110804Skan	.d_poll =	bpfpoll,
19050977Sjdp	.d_name =	"bpf",
19150977Sjdp	.d_kqfilter =	bpfkqfilter,
19250977Sjdp};
19350977Sjdp
19485004Sdfrstatic struct filterops bpfread_filtops = {
19534192Sjdp	.f_isfd = 1,
19634192Sjdp	.f_detach = filt_bpfdetach,
19734192Sjdp	.f_event = filt_bpfread,
19834192Sjdp};
19934192Sjdp
200133063Sdfr/*
201133063Sdfr * Wrapper functions for various buffering methods.  If the set of buffer
20276296Sjdp * modes expands, we will probably want to introduce a switch data structure
20376296Sjdp * similar to protosw, et.
20476296Sjdp */
20576296Sjdpstatic void
20676296Sjdpbpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
20776296Sjdp    u_int len)
20876296Sjdp{
20976296Sjdp
21076296Sjdp	BPFD_LOCK_ASSERT(d);
21148871Sjdp
21250609Sjdp	switch (d->bd_bufmode) {
21334192Sjdp	case BPF_BUFMODE_BUFFER:
21434192Sjdp		return (bpf_buffer_append_bytes(d, buf, offset, src, len));
21534192Sjdp
21638816Sdfr	case BPF_BUFMODE_ZBUF:
21734192Sjdp		d->bd_zcopy++;
218116563Smdodd		return (bpf_zerocopy_append_bytes(d, buf, offset, src, len));
219116563Smdodd
220116563Smdodd	default:
221116563Smdodd		panic("bpf_buf_append_bytes");
222116563Smdodd	}
22338816Sdfr}
22438816Sdfr
22538816Sdfrstatic void
22638816Sdfrbpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src,
22766056Sjdp    u_int len)
22876296Sjdp{
22945501Sjdp
230116557Smdodd	BPFD_LOCK_ASSERT(d);
23150608Sjdp
23250608Sjdp	switch (d->bd_bufmode) {
23345501Sjdp	case BPF_BUFMODE_BUFFER:
23438816Sdfr		return (bpf_buffer_append_mbuf(d, buf, offset, src, len));
23538816Sdfr
236133063Sdfr	case BPF_BUFMODE_ZBUF:
237133063Sdfr		d->bd_zcopy++;
238133063Sdfr		return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len));
239133063Sdfr
240133063Sdfr	default:
241142645Sdfr		panic("bpf_buf_append_mbuf");
24238816Sdfr	}
243116558Smdodd}
244116558Smdodd
245116558Smdodd/*
246116558Smdodd * This function gets called when the free buffer is re-assigned.
247116558Smdodd */
248116558Smdoddstatic void
249116558Smdoddbpf_buf_reclaimed(struct bpf_d *d)
250133063Sdfr{
251116558Smdodd
25234192Sjdp	BPFD_LOCK_ASSERT(d);
253
254	switch (d->bd_bufmode) {
255	case BPF_BUFMODE_BUFFER:
256		return;
257
258	case BPF_BUFMODE_ZBUF:
259		bpf_zerocopy_buf_reclaimed(d);
260		return;
261
262	default:
263		panic("bpf_buf_reclaimed");
264	}
265}
266
267/*
268 * If the buffer mechanism has a way to decide that a held buffer can be made
269 * free, then it is exposed via the bpf_canfreebuf() interface.  (1) is
270 * returned if the buffer can be discarded, (0) is returned if it cannot.
271 */
272static int
273bpf_canfreebuf(struct bpf_d *d)
274{
275
276	BPFD_LOCK_ASSERT(d);
277
278	switch (d->bd_bufmode) {
279	case BPF_BUFMODE_ZBUF:
280		return (bpf_zerocopy_canfreebuf(d));
281	}
282	return (0);
283}
284
285/*
286 * Allow the buffer model to indicate that the current store buffer is
287 * immutable, regardless of the appearance of space.  Return (1) if the
288 * buffer is writable, and (0) if not.
289 */
290static int
291bpf_canwritebuf(struct bpf_d *d)
292{
293
294	BPFD_LOCK_ASSERT(d);
295
296	switch (d->bd_bufmode) {
297	case BPF_BUFMODE_ZBUF:
298		return (bpf_zerocopy_canwritebuf(d));
299	}
300	return (1);
301}
302
303/*
304 * Notify buffer model that an attempt to write to the store buffer has
305 * resulted in a dropped packet, in which case the buffer may be considered
306 * full.
307 */
308static void
309bpf_buffull(struct bpf_d *d)
310{
311
312	BPFD_LOCK_ASSERT(d);
313
314	switch (d->bd_bufmode) {
315	case BPF_BUFMODE_ZBUF:
316		bpf_zerocopy_buffull(d);
317		break;
318	}
319}
320
321/*
322 * Notify the buffer model that a buffer has moved into the hold position.
323 */
324void
325bpf_bufheld(struct bpf_d *d)
326{
327
328	BPFD_LOCK_ASSERT(d);
329
330	switch (d->bd_bufmode) {
331	case BPF_BUFMODE_ZBUF:
332		bpf_zerocopy_bufheld(d);
333		break;
334	}
335}
336
337static void
338bpf_free(struct bpf_d *d)
339{
340
341	switch (d->bd_bufmode) {
342	case BPF_BUFMODE_BUFFER:
343		return (bpf_buffer_free(d));
344
345	case BPF_BUFMODE_ZBUF:
346		return (bpf_zerocopy_free(d));
347
348	default:
349		panic("bpf_buf_free");
350	}
351}
352
353static int
354bpf_uiomove(struct bpf_d *d, caddr_t buf, u_int len, struct uio *uio)
355{
356
357	if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
358		return (EOPNOTSUPP);
359	return (bpf_buffer_uiomove(d, buf, len, uio));
360}
361
362static int
363bpf_ioctl_sblen(struct bpf_d *d, u_int *i)
364{
365
366	if (d->bd_bufmode != BPF_BUFMODE_BUFFER)
367		return (EOPNOTSUPP);
368	return (bpf_buffer_ioctl_sblen(d, i));
369}
370
371static int
372bpf_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i)
373{
374
375	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
376		return (EOPNOTSUPP);
377	return (bpf_zerocopy_ioctl_getzmax(td, d, i));
378}
379
380static int
381bpf_ioctl_rotzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
382{
383
384	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
385		return (EOPNOTSUPP);
386	return (bpf_zerocopy_ioctl_rotzbuf(td, d, bz));
387}
388
389static int
390bpf_ioctl_setzbuf(struct thread *td, struct bpf_d *d, struct bpf_zbuf *bz)
391{
392
393	if (d->bd_bufmode != BPF_BUFMODE_ZBUF)
394		return (EOPNOTSUPP);
395	return (bpf_zerocopy_ioctl_setzbuf(td, d, bz));
396}
397
398/*
399 * General BPF functions.
400 */
401static int
402bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp,
403    struct sockaddr *sockp, int *hdrlen, struct bpf_insn *wfilter)
404{
405	const struct ieee80211_bpf_params *p;
406	struct ether_header *eh;
407	struct mbuf *m;
408	int error;
409	int len;
410	int hlen;
411	int slen;
412
413	/*
414	 * Build a sockaddr based on the data link layer type.
415	 * We do this at this level because the ethernet header
416	 * is copied directly into the data field of the sockaddr.
417	 * In the case of SLIP, there is no header and the packet
418	 * is forwarded as is.
419	 * Also, we are careful to leave room at the front of the mbuf
420	 * for the link level header.
421	 */
422	switch (linktype) {
423
424	case DLT_SLIP:
425		sockp->sa_family = AF_INET;
426		hlen = 0;
427		break;
428
429	case DLT_EN10MB:
430		sockp->sa_family = AF_UNSPEC;
431		/* XXX Would MAXLINKHDR be better? */
432		hlen = ETHER_HDR_LEN;
433		break;
434
435	case DLT_FDDI:
436		sockp->sa_family = AF_IMPLINK;
437		hlen = 0;
438		break;
439
440	case DLT_RAW:
441		sockp->sa_family = AF_UNSPEC;
442		hlen = 0;
443		break;
444
445	case DLT_NULL:
446		/*
447		 * null interface types require a 4 byte pseudo header which
448		 * corresponds to the address family of the packet.
449		 */
450		sockp->sa_family = AF_UNSPEC;
451		hlen = 4;
452		break;
453
454	case DLT_ATM_RFC1483:
455		/*
456		 * en atm driver requires 4-byte atm pseudo header.
457		 * though it isn't standard, vpi:vci needs to be
458		 * specified anyway.
459		 */
460		sockp->sa_family = AF_UNSPEC;
461		hlen = 12;	/* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
462		break;
463
464	case DLT_PPP:
465		sockp->sa_family = AF_UNSPEC;
466		hlen = 4;	/* This should match PPP_HDRLEN */
467		break;
468
469	case DLT_IEEE802_11:		/* IEEE 802.11 wireless */
470		sockp->sa_family = AF_IEEE80211;
471		hlen = 0;
472		break;
473
474	case DLT_IEEE802_11_RADIO:	/* IEEE 802.11 wireless w/ phy params */
475		sockp->sa_family = AF_IEEE80211;
476		sockp->sa_len = 12;	/* XXX != 0 */
477		hlen = sizeof(struct ieee80211_bpf_params);
478		break;
479
480	default:
481		return (EIO);
482	}
483
484	len = uio->uio_resid;
485
486	if (len - hlen > ifp->if_mtu)
487		return (EMSGSIZE);
488
489	if ((unsigned)len > MJUM16BYTES)
490		return (EIO);
491
492	if (len <= MHLEN)
493		MGETHDR(m, M_WAIT, MT_DATA);
494	else if (len <= MCLBYTES)
495		m = m_getcl(M_WAIT, MT_DATA, M_PKTHDR);
496	else
497		m = m_getjcl(M_WAIT, MT_DATA, M_PKTHDR,
498#if (MJUMPAGESIZE > MCLBYTES)
499		    len <= MJUMPAGESIZE ? MJUMPAGESIZE :
500#endif
501		    (len <= MJUM9BYTES ? MJUM9BYTES : MJUM16BYTES));
502	m->m_pkthdr.len = m->m_len = len;
503	m->m_pkthdr.rcvif = NULL;
504	*mp = m;
505
506	if (m->m_len < hlen) {
507		error = EPERM;
508		goto bad;
509	}
510
511	error = uiomove(mtod(m, u_char *), len, uio);
512	if (error)
513		goto bad;
514
515	slen = bpf_filter(wfilter, mtod(m, u_char *), len, len);
516	if (slen == 0) {
517		error = EPERM;
518		goto bad;
519	}
520
521	/* Check for multicast destination */
522	switch (linktype) {
523	case DLT_EN10MB:
524		eh = mtod(m, struct ether_header *);
525		if (ETHER_IS_MULTICAST(eh->ether_dhost)) {
526			if (bcmp(ifp->if_broadcastaddr, eh->ether_dhost,
527			    ETHER_ADDR_LEN) == 0)
528				m->m_flags |= M_BCAST;
529			else
530				m->m_flags |= M_MCAST;
531		}
532		break;
533	}
534
535	/*
536	 * Make room for link header, and copy it to sockaddr
537	 */
538	if (hlen != 0) {
539		if (sockp->sa_family == AF_IEEE80211) {
540			/*
541			 * Collect true length from the parameter header
542			 * NB: sockp is known to be zero'd so if we do a
543			 *     short copy unspecified parameters will be
544			 *     zero.
545			 * NB: packet may not be aligned after stripping
546			 *     bpf params
547			 * XXX check ibp_vers
548			 */
549			p = mtod(m, const struct ieee80211_bpf_params *);
550			hlen = p->ibp_len;
551			if (hlen > sizeof(sockp->sa_data)) {
552				error = EINVAL;
553				goto bad;
554			}
555		}
556		bcopy(m->m_data, sockp->sa_data, hlen);
557	}
558	*hdrlen = hlen;
559
560	return (0);
561bad:
562	m_freem(m);
563	return (error);
564}
565
566/*
567 * Attach file to the bpf interface, i.e. make d listen on bp.
568 */
569static void
570bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
571{
572	/*
573	 * Point d at bp, and add d to the interface's list of listeners.
574	 * Finally, point the driver's bpf cookie at the interface so
575	 * it will divert packets to bpf.
576	 */
577	BPFIF_LOCK(bp);
578	d->bd_bif = bp;
579	LIST_INSERT_HEAD(&bp->bif_dlist, d, bd_next);
580
581	bpf_bpfd_cnt++;
582	BPFIF_UNLOCK(bp);
583
584	EVENTHANDLER_INVOKE(bpf_track, bp->bif_ifp, bp->bif_dlt, 1);
585}
586
587/*
588 * Detach a file from its interface.
589 */
590static void
591bpf_detachd(struct bpf_d *d)
592{
593	int error;
594	struct bpf_if *bp;
595	struct ifnet *ifp;
596
597	bp = d->bd_bif;
598	BPFIF_LOCK(bp);
599	BPFD_LOCK(d);
600	ifp = d->bd_bif->bif_ifp;
601
602	/*
603	 * Remove d from the interface's descriptor list.
604	 */
605	LIST_REMOVE(d, bd_next);
606
607	bpf_bpfd_cnt--;
608	d->bd_bif = NULL;
609	BPFD_UNLOCK(d);
610	BPFIF_UNLOCK(bp);
611
612	EVENTHANDLER_INVOKE(bpf_track, ifp, bp->bif_dlt, 0);
613
614	/*
615	 * Check if this descriptor had requested promiscuous mode.
616	 * If so, turn it off.
617	 */
618	if (d->bd_promisc) {
619		d->bd_promisc = 0;
620		CURVNET_SET(ifp->if_vnet);
621		error = ifpromisc(ifp, 0);
622		CURVNET_RESTORE();
623		if (error != 0 && error != ENXIO) {
624			/*
625			 * ENXIO can happen if a pccard is unplugged
626			 * Something is really wrong if we were able to put
627			 * the driver into promiscuous mode, but can't
628			 * take it out.
629			 */
630			if_printf(bp->bif_ifp,
631				"bpf_detach: ifpromisc failed (%d)\n", error);
632		}
633	}
634}
635
636/*
637 * Close the descriptor by detaching it from its interface,
638 * deallocating its buffers, and marking it free.
639 */
640static void
641bpf_dtor(void *data)
642{
643	struct bpf_d *d = data;
644
645	BPFD_LOCK(d);
646	if (d->bd_state == BPF_WAITING)
647		callout_stop(&d->bd_callout);
648	d->bd_state = BPF_IDLE;
649	BPFD_UNLOCK(d);
650	funsetown(&d->bd_sigio);
651	mtx_lock(&bpf_mtx);
652	if (d->bd_bif)
653		bpf_detachd(d);
654	mtx_unlock(&bpf_mtx);
655#ifdef MAC
656	mac_bpfdesc_destroy(d);
657#endif /* MAC */
658	seldrain(&d->bd_sel);
659	knlist_destroy(&d->bd_sel.si_note);
660	callout_drain(&d->bd_callout);
661	bpf_freed(d);
662	free(d, M_BPF);
663}
664
665/*
666 * Open ethernet device.  Returns ENXIO for illegal minor device number,
667 * EBUSY if file is open by another process.
668 */
669/* ARGSUSED */
670static	int
671bpfopen(struct cdev *dev, int flags, int fmt, struct thread *td)
672{
673	struct bpf_d *d;
674	int error;
675
676	d = malloc(sizeof(*d), M_BPF, M_WAITOK | M_ZERO);
677	error = devfs_set_cdevpriv(d, bpf_dtor);
678	if (error != 0) {
679		free(d, M_BPF);
680		return (error);
681	}
682
683	/*
684	 * For historical reasons, perform a one-time initialization call to
685	 * the buffer routines, even though we're not yet committed to a
686	 * particular buffer method.
687	 */
688	bpf_buffer_init(d);
689	d->bd_bufmode = BPF_BUFMODE_BUFFER;
690	d->bd_sig = SIGIO;
691	d->bd_direction = BPF_D_INOUT;
692	d->bd_pid = td->td_proc->p_pid;
693#ifdef MAC
694	mac_bpfdesc_init(d);
695	mac_bpfdesc_create(td->td_ucred, d);
696#endif
697	mtx_init(&d->bd_mtx, devtoname(dev), "bpf cdev lock", MTX_DEF);
698	callout_init_mtx(&d->bd_callout, &d->bd_mtx, 0);
699	knlist_init_mtx(&d->bd_sel.si_note, &d->bd_mtx);
700
701	return (0);
702}
703
704/*
705 *  bpfread - read next chunk of packets from buffers
706 */
707static	int
708bpfread(struct cdev *dev, struct uio *uio, int ioflag)
709{
710	struct bpf_d *d;
711	int error;
712	int non_block;
713	int timed_out;
714
715	error = devfs_get_cdevpriv((void **)&d);
716	if (error != 0)
717		return (error);
718
719	/*
720	 * Restrict application to use a buffer the same size as
721	 * as kernel buffers.
722	 */
723	if (uio->uio_resid != d->bd_bufsize)
724		return (EINVAL);
725
726	non_block = ((ioflag & O_NONBLOCK) != 0);
727
728	BPFD_LOCK(d);
729	d->bd_pid = curthread->td_proc->p_pid;
730	if (d->bd_bufmode != BPF_BUFMODE_BUFFER) {
731		BPFD_UNLOCK(d);
732		return (EOPNOTSUPP);
733	}
734	if (d->bd_state == BPF_WAITING)
735		callout_stop(&d->bd_callout);
736	timed_out = (d->bd_state == BPF_TIMED_OUT);
737	d->bd_state = BPF_IDLE;
738	/*
739	 * If the hold buffer is empty, then do a timed sleep, which
740	 * ends when the timeout expires or when enough packets
741	 * have arrived to fill the store buffer.
742	 */
743	while (d->bd_hbuf == NULL) {
744		if (d->bd_slen != 0) {
745			/*
746			 * A packet(s) either arrived since the previous
747			 * read or arrived while we were asleep.
748			 */
749			if (d->bd_immediate || non_block || timed_out) {
750				/*
751				 * Rotate the buffers and return what's here
752				 * if we are in immediate mode, non-blocking
753				 * flag is set, or this descriptor timed out.
754				 */
755				ROTATE_BUFFERS(d);
756				break;
757			}
758		}
759
760		/*
761		 * No data is available, check to see if the bpf device
762		 * is still pointed at a real interface.  If not, return
763		 * ENXIO so that the userland process knows to rebind
764		 * it before using it again.
765		 */
766		if (d->bd_bif == NULL) {
767			BPFD_UNLOCK(d);
768			return (ENXIO);
769		}
770
771		if (non_block) {
772			BPFD_UNLOCK(d);
773			return (EWOULDBLOCK);
774		}
775		error = msleep(d, &d->bd_mtx, PRINET|PCATCH,
776		     "bpf", d->bd_rtout);
777		if (error == EINTR || error == ERESTART) {
778			BPFD_UNLOCK(d);
779			return (error);
780		}
781		if (error == EWOULDBLOCK) {
782			/*
783			 * On a timeout, return what's in the buffer,
784			 * which may be nothing.  If there is something
785			 * in the store buffer, we can rotate the buffers.
786			 */
787			if (d->bd_hbuf)
788				/*
789				 * We filled up the buffer in between
790				 * getting the timeout and arriving
791				 * here, so we don't need to rotate.
792				 */
793				break;
794
795			if (d->bd_slen == 0) {
796				BPFD_UNLOCK(d);
797				return (0);
798			}
799			ROTATE_BUFFERS(d);
800			break;
801		}
802	}
803	/*
804	 * At this point, we know we have something in the hold slot.
805	 */
806	BPFD_UNLOCK(d);
807
808	/*
809	 * Move data from hold buffer into user space.
810	 * We know the entire buffer is transferred since
811	 * we checked above that the read buffer is bpf_bufsize bytes.
812	 *
813	 * XXXRW: More synchronization needed here: what if a second thread
814	 * issues a read on the same fd at the same time?  Don't want this
815	 * getting invalidated.
816	 */
817	error = bpf_uiomove(d, d->bd_hbuf, d->bd_hlen, uio);
818
819	BPFD_LOCK(d);
820	d->bd_fbuf = d->bd_hbuf;
821	d->bd_hbuf = NULL;
822	d->bd_hlen = 0;
823	bpf_buf_reclaimed(d);
824	BPFD_UNLOCK(d);
825
826	return (error);
827}
828
829/*
830 * If there are processes sleeping on this descriptor, wake them up.
831 */
832static __inline void
833bpf_wakeup(struct bpf_d *d)
834{
835
836	BPFD_LOCK_ASSERT(d);
837	if (d->bd_state == BPF_WAITING) {
838		callout_stop(&d->bd_callout);
839		d->bd_state = BPF_IDLE;
840	}
841	wakeup(d);
842	if (d->bd_async && d->bd_sig && d->bd_sigio)
843		pgsigio(&d->bd_sigio, d->bd_sig, 0);
844
845	selwakeuppri(&d->bd_sel, PRINET);
846	KNOTE_LOCKED(&d->bd_sel.si_note, 0);
847}
848
849static void
850bpf_timed_out(void *arg)
851{
852	struct bpf_d *d = (struct bpf_d *)arg;
853
854	BPFD_LOCK_ASSERT(d);
855
856	if (callout_pending(&d->bd_callout) || !callout_active(&d->bd_callout))
857		return;
858	if (d->bd_state == BPF_WAITING) {
859		d->bd_state = BPF_TIMED_OUT;
860		if (d->bd_slen != 0)
861			bpf_wakeup(d);
862	}
863}
864
865static int
866bpf_ready(struct bpf_d *d)
867{
868
869	BPFD_LOCK_ASSERT(d);
870
871	if (!bpf_canfreebuf(d) && d->bd_hlen != 0)
872		return (1);
873	if ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) &&
874	    d->bd_slen != 0)
875		return (1);
876	return (0);
877}
878
879static int
880bpfwrite(struct cdev *dev, struct uio *uio, int ioflag)
881{
882	struct bpf_d *d;
883	struct ifnet *ifp;
884	struct mbuf *m, *mc;
885	struct sockaddr dst;
886	int error, hlen;
887
888	error = devfs_get_cdevpriv((void **)&d);
889	if (error != 0)
890		return (error);
891
892	d->bd_pid = curthread->td_proc->p_pid;
893	d->bd_wcount++;
894	if (d->bd_bif == NULL) {
895		d->bd_wdcount++;
896		return (ENXIO);
897	}
898
899	ifp = d->bd_bif->bif_ifp;
900
901	if ((ifp->if_flags & IFF_UP) == 0) {
902		d->bd_wdcount++;
903		return (ENETDOWN);
904	}
905
906	if (uio->uio_resid == 0) {
907		d->bd_wdcount++;
908		return (0);
909	}
910
911	bzero(&dst, sizeof(dst));
912	m = NULL;
913	hlen = 0;
914	error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp,
915	    &m, &dst, &hlen, d->bd_wfilter);
916	if (error) {
917		d->bd_wdcount++;
918		return (error);
919	}
920	d->bd_wfcount++;
921	if (d->bd_hdrcmplt)
922		dst.sa_family = pseudo_AF_HDRCMPLT;
923
924	if (d->bd_feedback) {
925		mc = m_dup(m, M_DONTWAIT);
926		if (mc != NULL)
927			mc->m_pkthdr.rcvif = ifp;
928		/* Set M_PROMISC for outgoing packets to be discarded. */
929		if (d->bd_direction == BPF_D_INOUT)
930			m->m_flags |= M_PROMISC;
931	} else
932		mc = NULL;
933
934	m->m_pkthdr.len -= hlen;
935	m->m_len -= hlen;
936	m->m_data += hlen;	/* XXX */
937
938	CURVNET_SET(ifp->if_vnet);
939#ifdef MAC
940	BPFD_LOCK(d);
941	mac_bpfdesc_create_mbuf(d, m);
942	if (mc != NULL)
943		mac_bpfdesc_create_mbuf(d, mc);
944	BPFD_UNLOCK(d);
945#endif
946
947	error = (*ifp->if_output)(ifp, m, &dst, NULL);
948	if (error)
949		d->bd_wdcount++;
950
951	if (mc != NULL) {
952		if (error == 0)
953			(*ifp->if_input)(ifp, mc);
954		else
955			m_freem(mc);
956	}
957	CURVNET_RESTORE();
958
959	return (error);
960}
961
962/*
963 * Reset a descriptor by flushing its packet buffer and clearing the receive
964 * and drop counts.  This is doable for kernel-only buffers, but with
965 * zero-copy buffers, we can't write to (or rotate) buffers that are
966 * currently owned by userspace.  It would be nice if we could encapsulate
967 * this logic in the buffer code rather than here.
968 */
969static void
970reset_d(struct bpf_d *d)
971{
972
973	mtx_assert(&d->bd_mtx, MA_OWNED);
974
975	if ((d->bd_hbuf != NULL) &&
976	    (d->bd_bufmode != BPF_BUFMODE_ZBUF || bpf_canfreebuf(d))) {
977		/* Free the hold buffer. */
978		d->bd_fbuf = d->bd_hbuf;
979		d->bd_hbuf = NULL;
980		d->bd_hlen = 0;
981		bpf_buf_reclaimed(d);
982	}
983	if (bpf_canwritebuf(d))
984		d->bd_slen = 0;
985	d->bd_rcount = 0;
986	d->bd_dcount = 0;
987	d->bd_fcount = 0;
988	d->bd_wcount = 0;
989	d->bd_wfcount = 0;
990	d->bd_wdcount = 0;
991	d->bd_zcopy = 0;
992}
993
994/*
995 *  FIONREAD		Check for read packet available.
996 *  SIOCGIFADDR		Get interface address - convenient hook to driver.
997 *  BIOCGBLEN		Get buffer len [for read()].
998 *  BIOCSETF		Set read filter.
999 *  BIOCSETFNR		Set read filter without resetting descriptor.
1000 *  BIOCSETWF		Set write filter.
1001 *  BIOCFLUSH		Flush read packet buffer.
1002 *  BIOCPROMISC		Put interface into promiscuous mode.
1003 *  BIOCGDLT		Get link layer type.
1004 *  BIOCGETIF		Get interface name.
1005 *  BIOCSETIF		Set interface.
1006 *  BIOCSRTIMEOUT	Set read timeout.
1007 *  BIOCGRTIMEOUT	Get read timeout.
1008 *  BIOCGSTATS		Get packet stats.
1009 *  BIOCIMMEDIATE	Set immediate mode.
1010 *  BIOCVERSION		Get filter language version.
1011 *  BIOCGHDRCMPLT	Get "header already complete" flag
1012 *  BIOCSHDRCMPLT	Set "header already complete" flag
1013 *  BIOCGDIRECTION	Get packet direction flag
1014 *  BIOCSDIRECTION	Set packet direction flag
1015 *  BIOCGTSTAMP		Get time stamp format and resolution.
1016 *  BIOCSTSTAMP		Set time stamp format and resolution.
1017 *  BIOCLOCK		Set "locked" flag
1018 *  BIOCFEEDBACK	Set packet feedback mode.
1019 *  BIOCSETZBUF		Set current zero-copy buffer locations.
1020 *  BIOCGETZMAX		Get maximum zero-copy buffer size.
1021 *  BIOCROTZBUF		Force rotation of zero-copy buffer
1022 *  BIOCSETBUFMODE	Set buffer mode.
1023 *  BIOCGETBUFMODE	Get current buffer mode.
1024 */
1025/* ARGSUSED */
1026static	int
1027bpfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags,
1028    struct thread *td)
1029{
1030	struct bpf_d *d;
1031	int error;
1032
1033	error = devfs_get_cdevpriv((void **)&d);
1034	if (error != 0)
1035		return (error);
1036
1037	/*
1038	 * Refresh PID associated with this descriptor.
1039	 */
1040	BPFD_LOCK(d);
1041	d->bd_pid = td->td_proc->p_pid;
1042	if (d->bd_state == BPF_WAITING)
1043		callout_stop(&d->bd_callout);
1044	d->bd_state = BPF_IDLE;
1045	BPFD_UNLOCK(d);
1046
1047	if (d->bd_locked == 1) {
1048		switch (cmd) {
1049		case BIOCGBLEN:
1050		case BIOCFLUSH:
1051		case BIOCGDLT:
1052		case BIOCGDLTLIST:
1053#ifdef COMPAT_FREEBSD32
1054		case BIOCGDLTLIST32:
1055#endif
1056		case BIOCGETIF:
1057		case BIOCGRTIMEOUT:
1058#ifdef COMPAT_FREEBSD32
1059		case BIOCGRTIMEOUT32:
1060#endif
1061		case BIOCGSTATS:
1062		case BIOCVERSION:
1063		case BIOCGRSIG:
1064		case BIOCGHDRCMPLT:
1065		case BIOCSTSTAMP:
1066		case BIOCFEEDBACK:
1067		case FIONREAD:
1068		case BIOCLOCK:
1069		case BIOCSRTIMEOUT:
1070#ifdef COMPAT_FREEBSD32
1071		case BIOCSRTIMEOUT32:
1072#endif
1073		case BIOCIMMEDIATE:
1074		case TIOCGPGRP:
1075		case BIOCROTZBUF:
1076			break;
1077		default:
1078			return (EPERM);
1079		}
1080	}
1081#ifdef COMPAT_FREEBSD32
1082	/*
1083	 * If we see a 32-bit compat ioctl, mark the stream as 32-bit so
1084	 * that it will get 32-bit packet headers.
1085	 */
1086	switch (cmd) {
1087	case BIOCSETF32:
1088	case BIOCSETFNR32:
1089	case BIOCSETWF32:
1090	case BIOCGDLTLIST32:
1091	case BIOCGRTIMEOUT32:
1092	case BIOCSRTIMEOUT32:
1093		d->bd_compat32 = 1;
1094	}
1095#endif
1096
1097	CURVNET_SET(TD_TO_VNET(td));
1098	switch (cmd) {
1099
1100	default:
1101		error = EINVAL;
1102		break;
1103
1104	/*
1105	 * Check for read packet available.
1106	 */
1107	case FIONREAD:
1108		{
1109			int n;
1110
1111			BPFD_LOCK(d);
1112			n = d->bd_slen;
1113			if (d->bd_hbuf)
1114				n += d->bd_hlen;
1115			BPFD_UNLOCK(d);
1116
1117			*(int *)addr = n;
1118			break;
1119		}
1120
1121	case SIOCGIFADDR:
1122		{
1123			struct ifnet *ifp;
1124
1125			if (d->bd_bif == NULL)
1126				error = EINVAL;
1127			else {
1128				ifp = d->bd_bif->bif_ifp;
1129				error = (*ifp->if_ioctl)(ifp, cmd, addr);
1130			}
1131			break;
1132		}
1133
1134	/*
1135	 * Get buffer len [for read()].
1136	 */
1137	case BIOCGBLEN:
1138		*(u_int *)addr = d->bd_bufsize;
1139		break;
1140
1141	/*
1142	 * Set buffer length.
1143	 */
1144	case BIOCSBLEN:
1145		error = bpf_ioctl_sblen(d, (u_int *)addr);
1146		break;
1147
1148	/*
1149	 * Set link layer read filter.
1150	 */
1151	case BIOCSETF:
1152	case BIOCSETFNR:
1153	case BIOCSETWF:
1154#ifdef COMPAT_FREEBSD32
1155	case BIOCSETF32:
1156	case BIOCSETFNR32:
1157	case BIOCSETWF32:
1158#endif
1159		error = bpf_setf(d, (struct bpf_program *)addr, cmd);
1160		break;
1161
1162	/*
1163	 * Flush read packet buffer.
1164	 */
1165	case BIOCFLUSH:
1166		BPFD_LOCK(d);
1167		reset_d(d);
1168		BPFD_UNLOCK(d);
1169		break;
1170
1171	/*
1172	 * Put interface into promiscuous mode.
1173	 */
1174	case BIOCPROMISC:
1175		if (d->bd_bif == NULL) {
1176			/*
1177			 * No interface attached yet.
1178			 */
1179			error = EINVAL;
1180			break;
1181		}
1182		if (d->bd_promisc == 0) {
1183			error = ifpromisc(d->bd_bif->bif_ifp, 1);
1184			if (error == 0)
1185				d->bd_promisc = 1;
1186		}
1187		break;
1188
1189	/*
1190	 * Get current data link type.
1191	 */
1192	case BIOCGDLT:
1193		if (d->bd_bif == NULL)
1194			error = EINVAL;
1195		else
1196			*(u_int *)addr = d->bd_bif->bif_dlt;
1197		break;
1198
1199	/*
1200	 * Get a list of supported data link types.
1201	 */
1202#ifdef COMPAT_FREEBSD32
1203	case BIOCGDLTLIST32:
1204		{
1205			struct bpf_dltlist32 *list32;
1206			struct bpf_dltlist dltlist;
1207
1208			list32 = (struct bpf_dltlist32 *)addr;
1209			dltlist.bfl_len = list32->bfl_len;
1210			dltlist.bfl_list = PTRIN(list32->bfl_list);
1211			if (d->bd_bif == NULL)
1212				error = EINVAL;
1213			else {
1214				error = bpf_getdltlist(d, &dltlist);
1215				if (error == 0)
1216					list32->bfl_len = dltlist.bfl_len;
1217			}
1218			break;
1219		}
1220#endif
1221
1222	case BIOCGDLTLIST:
1223		if (d->bd_bif == NULL)
1224			error = EINVAL;
1225		else
1226			error = bpf_getdltlist(d, (struct bpf_dltlist *)addr);
1227		break;
1228
1229	/*
1230	 * Set data link type.
1231	 */
1232	case BIOCSDLT:
1233		if (d->bd_bif == NULL)
1234			error = EINVAL;
1235		else
1236			error = bpf_setdlt(d, *(u_int *)addr);
1237		break;
1238
1239	/*
1240	 * Get interface name.
1241	 */
1242	case BIOCGETIF:
1243		if (d->bd_bif == NULL)
1244			error = EINVAL;
1245		else {
1246			struct ifnet *const ifp = d->bd_bif->bif_ifp;
1247			struct ifreq *const ifr = (struct ifreq *)addr;
1248
1249			strlcpy(ifr->ifr_name, ifp->if_xname,
1250			    sizeof(ifr->ifr_name));
1251		}
1252		break;
1253
1254	/*
1255	 * Set interface.
1256	 */
1257	case BIOCSETIF:
1258		error = bpf_setif(d, (struct ifreq *)addr);
1259		break;
1260
1261	/*
1262	 * Set read timeout.
1263	 */
1264	case BIOCSRTIMEOUT:
1265#ifdef COMPAT_FREEBSD32
1266	case BIOCSRTIMEOUT32:
1267#endif
1268		{
1269			struct timeval *tv = (struct timeval *)addr;
1270#ifdef COMPAT_FREEBSD32
1271			struct timeval32 *tv32;
1272			struct timeval tv64;
1273
1274			if (cmd == BIOCSRTIMEOUT32) {
1275				tv32 = (struct timeval32 *)addr;
1276				tv = &tv64;
1277				tv->tv_sec = tv32->tv_sec;
1278				tv->tv_usec = tv32->tv_usec;
1279			} else
1280#endif
1281				tv = (struct timeval *)addr;
1282
1283			/*
1284			 * Subtract 1 tick from tvtohz() since this isn't
1285			 * a one-shot timer.
1286			 */
1287			if ((error = itimerfix(tv)) == 0)
1288				d->bd_rtout = tvtohz(tv) - 1;
1289			break;
1290		}
1291
1292	/*
1293	 * Get read timeout.
1294	 */
1295	case BIOCGRTIMEOUT:
1296#ifdef COMPAT_FREEBSD32
1297	case BIOCGRTIMEOUT32:
1298#endif
1299		{
1300			struct timeval *tv;
1301#ifdef COMPAT_FREEBSD32
1302			struct timeval32 *tv32;
1303			struct timeval tv64;
1304
1305			if (cmd == BIOCGRTIMEOUT32)
1306				tv = &tv64;
1307			else
1308#endif
1309				tv = (struct timeval *)addr;
1310
1311			tv->tv_sec = d->bd_rtout / hz;
1312			tv->tv_usec = (d->bd_rtout % hz) * tick;
1313#ifdef COMPAT_FREEBSD32
1314			if (cmd == BIOCGRTIMEOUT32) {
1315				tv32 = (struct timeval32 *)addr;
1316				tv32->tv_sec = tv->tv_sec;
1317				tv32->tv_usec = tv->tv_usec;
1318			}
1319#endif
1320
1321			break;
1322		}
1323
1324	/*
1325	 * Get packet stats.
1326	 */
1327	case BIOCGSTATS:
1328		{
1329			struct bpf_stat *bs = (struct bpf_stat *)addr;
1330
1331			/* XXXCSJP overflow */
1332			bs->bs_recv = d->bd_rcount;
1333			bs->bs_drop = d->bd_dcount;
1334			break;
1335		}
1336
1337	/*
1338	 * Set immediate mode.
1339	 */
1340	case BIOCIMMEDIATE:
1341		d->bd_immediate = *(u_int *)addr;
1342		break;
1343
1344	case BIOCVERSION:
1345		{
1346			struct bpf_version *bv = (struct bpf_version *)addr;
1347
1348			bv->bv_major = BPF_MAJOR_VERSION;
1349			bv->bv_minor = BPF_MINOR_VERSION;
1350			break;
1351		}
1352
1353	/*
1354	 * Get "header already complete" flag
1355	 */
1356	case BIOCGHDRCMPLT:
1357		*(u_int *)addr = d->bd_hdrcmplt;
1358		break;
1359
1360	/*
1361	 * Set "header already complete" flag
1362	 */
1363	case BIOCSHDRCMPLT:
1364		d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0;
1365		break;
1366
1367	/*
1368	 * Get packet direction flag
1369	 */
1370	case BIOCGDIRECTION:
1371		*(u_int *)addr = d->bd_direction;
1372		break;
1373
1374	/*
1375	 * Set packet direction flag
1376	 */
1377	case BIOCSDIRECTION:
1378		{
1379			u_int	direction;
1380
1381			direction = *(u_int *)addr;
1382			switch (direction) {
1383			case BPF_D_IN:
1384			case BPF_D_INOUT:
1385			case BPF_D_OUT:
1386				d->bd_direction = direction;
1387				break;
1388			default:
1389				error = EINVAL;
1390			}
1391		}
1392		break;
1393
1394	/*
1395	 * Get packet timestamp format and resolution.
1396	 */
1397	case BIOCGTSTAMP:
1398		*(u_int *)addr = d->bd_tstamp;
1399		break;
1400
1401	/*
1402	 * Set packet timestamp format and resolution.
1403	 */
1404	case BIOCSTSTAMP:
1405		{
1406			u_int	func;
1407
1408			func = *(u_int *)addr;
1409			if (BPF_T_VALID(func))
1410				d->bd_tstamp = func;
1411			else
1412				error = EINVAL;
1413		}
1414		break;
1415
1416	case BIOCFEEDBACK:
1417		d->bd_feedback = *(u_int *)addr;
1418		break;
1419
1420	case BIOCLOCK:
1421		d->bd_locked = 1;
1422		break;
1423
1424	case FIONBIO:		/* Non-blocking I/O */
1425		break;
1426
1427	case FIOASYNC:		/* Send signal on receive packets */
1428		d->bd_async = *(int *)addr;
1429		break;
1430
1431	case FIOSETOWN:
1432		error = fsetown(*(int *)addr, &d->bd_sigio);
1433		break;
1434
1435	case FIOGETOWN:
1436		*(int *)addr = fgetown(&d->bd_sigio);
1437		break;
1438
1439	/* This is deprecated, FIOSETOWN should be used instead. */
1440	case TIOCSPGRP:
1441		error = fsetown(-(*(int *)addr), &d->bd_sigio);
1442		break;
1443
1444	/* This is deprecated, FIOGETOWN should be used instead. */
1445	case TIOCGPGRP:
1446		*(int *)addr = -fgetown(&d->bd_sigio);
1447		break;
1448
1449	case BIOCSRSIG:		/* Set receive signal */
1450		{
1451			u_int sig;
1452
1453			sig = *(u_int *)addr;
1454
1455			if (sig >= NSIG)
1456				error = EINVAL;
1457			else
1458				d->bd_sig = sig;
1459			break;
1460		}
1461	case BIOCGRSIG:
1462		*(u_int *)addr = d->bd_sig;
1463		break;
1464
1465	case BIOCGETBUFMODE:
1466		*(u_int *)addr = d->bd_bufmode;
1467		break;
1468
1469	case BIOCSETBUFMODE:
1470		/*
1471		 * Allow the buffering mode to be changed as long as we
1472		 * haven't yet committed to a particular mode.  Our
1473		 * definition of commitment, for now, is whether or not a
1474		 * buffer has been allocated or an interface attached, since
1475		 * that's the point where things get tricky.
1476		 */
1477		switch (*(u_int *)addr) {
1478		case BPF_BUFMODE_BUFFER:
1479			break;
1480
1481		case BPF_BUFMODE_ZBUF:
1482			if (bpf_zerocopy_enable)
1483				break;
1484			/* FALLSTHROUGH */
1485
1486		default:
1487			CURVNET_RESTORE();
1488			return (EINVAL);
1489		}
1490
1491		BPFD_LOCK(d);
1492		if (d->bd_sbuf != NULL || d->bd_hbuf != NULL ||
1493		    d->bd_fbuf != NULL || d->bd_bif != NULL) {
1494			BPFD_UNLOCK(d);
1495			CURVNET_RESTORE();
1496			return (EBUSY);
1497		}
1498		d->bd_bufmode = *(u_int *)addr;
1499		BPFD_UNLOCK(d);
1500		break;
1501
1502	case BIOCGETZMAX:
1503		error = bpf_ioctl_getzmax(td, d, (size_t *)addr);
1504		break;
1505
1506	case BIOCSETZBUF:
1507		error = bpf_ioctl_setzbuf(td, d, (struct bpf_zbuf *)addr);
1508		break;
1509
1510	case BIOCROTZBUF:
1511		error = bpf_ioctl_rotzbuf(td, d, (struct bpf_zbuf *)addr);
1512		break;
1513	}
1514	CURVNET_RESTORE();
1515	return (error);
1516}
1517
1518/*
1519 * Set d's packet filter program to fp.  If this file already has a filter,
1520 * free it and replace it.  Returns EINVAL for bogus requests.
1521 */
1522static int
1523bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd)
1524{
1525	struct bpf_insn *fcode, *old;
1526	u_int wfilter, flen, size;
1527#ifdef BPF_JITTER
1528	bpf_jit_filter *ofunc;
1529#endif
1530#ifdef COMPAT_FREEBSD32
1531	struct bpf_program32 *fp32;
1532	struct bpf_program fp_swab;
1533
1534	if (cmd == BIOCSETWF32 || cmd == BIOCSETF32 || cmd == BIOCSETFNR32) {
1535		fp32 = (struct bpf_program32 *)fp;
1536		fp_swab.bf_len = fp32->bf_len;
1537		fp_swab.bf_insns = (struct bpf_insn *)(uintptr_t)fp32->bf_insns;
1538		fp = &fp_swab;
1539		if (cmd == BIOCSETWF32)
1540			cmd = BIOCSETWF;
1541	}
1542#endif
1543	if (cmd == BIOCSETWF) {
1544		old = d->bd_wfilter;
1545		wfilter = 1;
1546#ifdef BPF_JITTER
1547		ofunc = NULL;
1548#endif
1549	} else {
1550		wfilter = 0;
1551		old = d->bd_rfilter;
1552#ifdef BPF_JITTER
1553		ofunc = d->bd_bfilter;
1554#endif
1555	}
1556	if (fp->bf_insns == NULL) {
1557		if (fp->bf_len != 0)
1558			return (EINVAL);
1559		BPFD_LOCK(d);
1560		if (wfilter)
1561			d->bd_wfilter = NULL;
1562		else {
1563			d->bd_rfilter = NULL;
1564#ifdef BPF_JITTER
1565			d->bd_bfilter = NULL;
1566#endif
1567			if (cmd == BIOCSETF)
1568				reset_d(d);
1569		}
1570		BPFD_UNLOCK(d);
1571		if (old != NULL)
1572			free((caddr_t)old, M_BPF);
1573#ifdef BPF_JITTER
1574		if (ofunc != NULL)
1575			bpf_destroy_jit_filter(ofunc);
1576#endif
1577		return (0);
1578	}
1579	flen = fp->bf_len;
1580	if (flen > bpf_maxinsns)
1581		return (EINVAL);
1582
1583	size = flen * sizeof(*fp->bf_insns);
1584	fcode = (struct bpf_insn *)malloc(size, M_BPF, M_WAITOK);
1585	if (copyin((caddr_t)fp->bf_insns, (caddr_t)fcode, size) == 0 &&
1586	    bpf_validate(fcode, (int)flen)) {
1587		BPFD_LOCK(d);
1588		if (wfilter)
1589			d->bd_wfilter = fcode;
1590		else {
1591			d->bd_rfilter = fcode;
1592#ifdef BPF_JITTER
1593			d->bd_bfilter = bpf_jitter(fcode, flen);
1594#endif
1595			if (cmd == BIOCSETF)
1596				reset_d(d);
1597		}
1598		BPFD_UNLOCK(d);
1599		if (old != NULL)
1600			free((caddr_t)old, M_BPF);
1601#ifdef BPF_JITTER
1602		if (ofunc != NULL)
1603			bpf_destroy_jit_filter(ofunc);
1604#endif
1605
1606		return (0);
1607	}
1608	free((caddr_t)fcode, M_BPF);
1609	return (EINVAL);
1610}
1611
1612/*
1613 * Detach a file from its current interface (if attached at all) and attach
1614 * to the interface indicated by the name stored in ifr.
1615 * Return an errno or 0.
1616 */
1617static int
1618bpf_setif(struct bpf_d *d, struct ifreq *ifr)
1619{
1620	struct bpf_if *bp;
1621	struct ifnet *theywant;
1622
1623	theywant = ifunit(ifr->ifr_name);
1624	if (theywant == NULL || theywant->if_bpf == NULL)
1625		return (ENXIO);
1626
1627	bp = theywant->if_bpf;
1628
1629	/*
1630	 * Behavior here depends on the buffering model.  If we're using
1631	 * kernel memory buffers, then we can allocate them here.  If we're
1632	 * using zero-copy, then the user process must have registered
1633	 * buffers by the time we get here.  If not, return an error.
1634	 *
1635	 * XXXRW: There are locking issues here with multi-threaded use: what
1636	 * if two threads try to set the interface at once?
1637	 */
1638	switch (d->bd_bufmode) {
1639	case BPF_BUFMODE_BUFFER:
1640		if (d->bd_sbuf == NULL)
1641			bpf_buffer_alloc(d);
1642		KASSERT(d->bd_sbuf != NULL, ("bpf_setif: bd_sbuf NULL"));
1643		break;
1644
1645	case BPF_BUFMODE_ZBUF:
1646		if (d->bd_sbuf == NULL)
1647			return (EINVAL);
1648		break;
1649
1650	default:
1651		panic("bpf_setif: bufmode %d", d->bd_bufmode);
1652	}
1653	if (bp != d->bd_bif) {
1654		if (d->bd_bif)
1655			/*
1656			 * Detach if attached to something else.
1657			 */
1658			bpf_detachd(d);
1659
1660		bpf_attachd(d, bp);
1661	}
1662	BPFD_LOCK(d);
1663	reset_d(d);
1664	BPFD_UNLOCK(d);
1665	return (0);
1666}
1667
1668/*
1669 * Support for select() and poll() system calls
1670 *
1671 * Return true iff the specific operation will not block indefinitely.
1672 * Otherwise, return false but make a note that a selwakeup() must be done.
1673 */
1674static int
1675bpfpoll(struct cdev *dev, int events, struct thread *td)
1676{
1677	struct bpf_d *d;
1678	int revents;
1679
1680	if (devfs_get_cdevpriv((void **)&d) != 0 || d->bd_bif == NULL)
1681		return (events &
1682		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1683
1684	/*
1685	 * Refresh PID associated with this descriptor.
1686	 */
1687	revents = events & (POLLOUT | POLLWRNORM);
1688	BPFD_LOCK(d);
1689	d->bd_pid = td->td_proc->p_pid;
1690	if (events & (POLLIN | POLLRDNORM)) {
1691		if (bpf_ready(d))
1692			revents |= events & (POLLIN | POLLRDNORM);
1693		else {
1694			selrecord(td, &d->bd_sel);
1695			/* Start the read timeout if necessary. */
1696			if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1697				callout_reset(&d->bd_callout, d->bd_rtout,
1698				    bpf_timed_out, d);
1699				d->bd_state = BPF_WAITING;
1700			}
1701		}
1702	}
1703	BPFD_UNLOCK(d);
1704	return (revents);
1705}
1706
1707/*
1708 * Support for kevent() system call.  Register EVFILT_READ filters and
1709 * reject all others.
1710 */
1711int
1712bpfkqfilter(struct cdev *dev, struct knote *kn)
1713{
1714	struct bpf_d *d;
1715
1716	if (devfs_get_cdevpriv((void **)&d) != 0 ||
1717	    kn->kn_filter != EVFILT_READ)
1718		return (1);
1719
1720	/*
1721	 * Refresh PID associated with this descriptor.
1722	 */
1723	BPFD_LOCK(d);
1724	d->bd_pid = curthread->td_proc->p_pid;
1725	kn->kn_fop = &bpfread_filtops;
1726	kn->kn_hook = d;
1727	knlist_add(&d->bd_sel.si_note, kn, 1);
1728	BPFD_UNLOCK(d);
1729
1730	return (0);
1731}
1732
1733static void
1734filt_bpfdetach(struct knote *kn)
1735{
1736	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
1737
1738	knlist_remove(&d->bd_sel.si_note, kn, 0);
1739}
1740
1741static int
1742filt_bpfread(struct knote *kn, long hint)
1743{
1744	struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
1745	int ready;
1746
1747	BPFD_LOCK_ASSERT(d);
1748	ready = bpf_ready(d);
1749	if (ready) {
1750		kn->kn_data = d->bd_slen;
1751		if (d->bd_hbuf)
1752			kn->kn_data += d->bd_hlen;
1753	} else if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
1754		callout_reset(&d->bd_callout, d->bd_rtout,
1755		    bpf_timed_out, d);
1756		d->bd_state = BPF_WAITING;
1757	}
1758
1759	return (ready);
1760}
1761
1762#define	BPF_TSTAMP_NONE		0
1763#define	BPF_TSTAMP_FAST		1
1764#define	BPF_TSTAMP_NORMAL	2
1765#define	BPF_TSTAMP_EXTERN	3
1766
1767static int
1768bpf_ts_quality(int tstype)
1769{
1770
1771	if (tstype == BPF_T_NONE)
1772		return (BPF_TSTAMP_NONE);
1773	if ((tstype & BPF_T_FAST) != 0)
1774		return (BPF_TSTAMP_FAST);
1775
1776	return (BPF_TSTAMP_NORMAL);
1777}
1778
1779static int
1780bpf_gettime(struct bintime *bt, int tstype, struct mbuf *m)
1781{
1782	struct m_tag *tag;
1783	int quality;
1784
1785	quality = bpf_ts_quality(tstype);
1786	if (quality == BPF_TSTAMP_NONE)
1787		return (quality);
1788
1789	if (m != NULL) {
1790		tag = m_tag_locate(m, MTAG_BPF, MTAG_BPF_TIMESTAMP, NULL);
1791		if (tag != NULL) {
1792			*bt = *(struct bintime *)(tag + 1);
1793			return (BPF_TSTAMP_EXTERN);
1794		}
1795	}
1796	if (quality == BPF_TSTAMP_NORMAL)
1797		binuptime(bt);
1798	else
1799		getbinuptime(bt);
1800
1801	return (quality);
1802}
1803
1804/*
1805 * Incoming linkage from device drivers.  Process the packet pkt, of length
1806 * pktlen, which is stored in a contiguous buffer.  The packet is parsed
1807 * by each process' filter, and if accepted, stashed into the corresponding
1808 * buffer.
1809 */
1810void
1811bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
1812{
1813	struct bintime bt;
1814	struct bpf_d *d;
1815#ifdef BPF_JITTER
1816	bpf_jit_filter *bf;
1817#endif
1818	u_int slen;
1819	int gottime;
1820
1821	gottime = BPF_TSTAMP_NONE;
1822	BPFIF_LOCK(bp);
1823	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1824		BPFD_LOCK(d);
1825		++d->bd_rcount;
1826		/*
1827		 * NB: We dont call BPF_CHECK_DIRECTION() here since there is no
1828		 * way for the caller to indiciate to us whether this packet
1829		 * is inbound or outbound.  In the bpf_mtap() routines, we use
1830		 * the interface pointers on the mbuf to figure it out.
1831		 */
1832#ifdef BPF_JITTER
1833		bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL;
1834		if (bf != NULL)
1835			slen = (*(bf->func))(pkt, pktlen, pktlen);
1836		else
1837#endif
1838		slen = bpf_filter(d->bd_rfilter, pkt, pktlen, pktlen);
1839		if (slen != 0) {
1840			d->bd_fcount++;
1841			if (gottime < bpf_ts_quality(d->bd_tstamp))
1842				gottime = bpf_gettime(&bt, d->bd_tstamp, NULL);
1843#ifdef MAC
1844			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
1845#endif
1846				catchpacket(d, pkt, pktlen, slen,
1847				    bpf_append_bytes, &bt);
1848		}
1849		BPFD_UNLOCK(d);
1850	}
1851	BPFIF_UNLOCK(bp);
1852}
1853
1854#define	BPF_CHECK_DIRECTION(d, r, i)				\
1855	    (((d)->bd_direction == BPF_D_IN && (r) != (i)) ||	\
1856	    ((d)->bd_direction == BPF_D_OUT && (r) == (i)))
1857
1858/*
1859 * Incoming linkage from device drivers, when packet is in an mbuf chain.
1860 */
1861void
1862bpf_mtap(struct bpf_if *bp, struct mbuf *m)
1863{
1864	struct bintime bt;
1865	struct bpf_d *d;
1866#ifdef BPF_JITTER
1867	bpf_jit_filter *bf;
1868#endif
1869	u_int pktlen, slen;
1870	int gottime;
1871
1872	/* Skip outgoing duplicate packets. */
1873	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
1874		m->m_flags &= ~M_PROMISC;
1875		return;
1876	}
1877
1878	pktlen = m_length(m, NULL);
1879
1880	gottime = BPF_TSTAMP_NONE;
1881	BPFIF_LOCK(bp);
1882	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1883		if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp))
1884			continue;
1885		BPFD_LOCK(d);
1886		++d->bd_rcount;
1887#ifdef BPF_JITTER
1888		bf = bpf_jitter_enable != 0 ? d->bd_bfilter : NULL;
1889		/* XXX We cannot handle multiple mbufs. */
1890		if (bf != NULL && m->m_next == NULL)
1891			slen = (*(bf->func))(mtod(m, u_char *), pktlen, pktlen);
1892		else
1893#endif
1894		slen = bpf_filter(d->bd_rfilter, (u_char *)m, pktlen, 0);
1895		if (slen != 0) {
1896			d->bd_fcount++;
1897			if (gottime < bpf_ts_quality(d->bd_tstamp))
1898				gottime = bpf_gettime(&bt, d->bd_tstamp, m);
1899#ifdef MAC
1900			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
1901#endif
1902				catchpacket(d, (u_char *)m, pktlen, slen,
1903				    bpf_append_mbuf, &bt);
1904		}
1905		BPFD_UNLOCK(d);
1906	}
1907	BPFIF_UNLOCK(bp);
1908}
1909
1910/*
1911 * Incoming linkage from device drivers, when packet is in
1912 * an mbuf chain and to be prepended by a contiguous header.
1913 */
1914void
1915bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m)
1916{
1917	struct bintime bt;
1918	struct mbuf mb;
1919	struct bpf_d *d;
1920	u_int pktlen, slen;
1921	int gottime;
1922
1923	/* Skip outgoing duplicate packets. */
1924	if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) {
1925		m->m_flags &= ~M_PROMISC;
1926		return;
1927	}
1928
1929	pktlen = m_length(m, NULL);
1930	/*
1931	 * Craft on-stack mbuf suitable for passing to bpf_filter.
1932	 * Note that we cut corners here; we only setup what's
1933	 * absolutely needed--this mbuf should never go anywhere else.
1934	 */
1935	mb.m_next = m;
1936	mb.m_data = data;
1937	mb.m_len = dlen;
1938	pktlen += dlen;
1939
1940	gottime = BPF_TSTAMP_NONE;
1941	BPFIF_LOCK(bp);
1942	LIST_FOREACH(d, &bp->bif_dlist, bd_next) {
1943		if (BPF_CHECK_DIRECTION(d, m->m_pkthdr.rcvif, bp->bif_ifp))
1944			continue;
1945		BPFD_LOCK(d);
1946		++d->bd_rcount;
1947		slen = bpf_filter(d->bd_rfilter, (u_char *)&mb, pktlen, 0);
1948		if (slen != 0) {
1949			d->bd_fcount++;
1950			if (gottime < bpf_ts_quality(d->bd_tstamp))
1951				gottime = bpf_gettime(&bt, d->bd_tstamp, m);
1952#ifdef MAC
1953			if (mac_bpfdesc_check_receive(d, bp->bif_ifp) == 0)
1954#endif
1955				catchpacket(d, (u_char *)&mb, pktlen, slen,
1956				    bpf_append_mbuf, &bt);
1957		}
1958		BPFD_UNLOCK(d);
1959	}
1960	BPFIF_UNLOCK(bp);
1961}
1962
1963#undef	BPF_CHECK_DIRECTION
1964
1965#undef	BPF_TSTAMP_NONE
1966#undef	BPF_TSTAMP_FAST
1967#undef	BPF_TSTAMP_NORMAL
1968#undef	BPF_TSTAMP_EXTERN
1969
1970static int
1971bpf_hdrlen(struct bpf_d *d)
1972{
1973	int hdrlen;
1974
1975	hdrlen = d->bd_bif->bif_hdrlen;
1976#ifndef BURN_BRIDGES
1977	if (d->bd_tstamp == BPF_T_NONE ||
1978	    BPF_T_FORMAT(d->bd_tstamp) == BPF_T_MICROTIME)
1979#ifdef COMPAT_FREEBSD32
1980		if (d->bd_compat32)
1981			hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr32);
1982		else
1983#endif
1984			hdrlen += SIZEOF_BPF_HDR(struct bpf_hdr);
1985	else
1986#endif
1987		hdrlen += SIZEOF_BPF_HDR(struct bpf_xhdr);
1988#ifdef COMPAT_FREEBSD32
1989	if (d->bd_compat32)
1990		hdrlen = BPF_WORDALIGN32(hdrlen);
1991	else
1992#endif
1993		hdrlen = BPF_WORDALIGN(hdrlen);
1994
1995	return (hdrlen - d->bd_bif->bif_hdrlen);
1996}
1997
1998static void
1999bpf_bintime2ts(struct bintime *bt, struct bpf_ts *ts, int tstype)
2000{
2001	struct bintime bt2;
2002	struct timeval tsm;
2003	struct timespec tsn;
2004
2005	if ((tstype & BPF_T_MONOTONIC) == 0) {
2006		bt2 = *bt;
2007		bintime_add(&bt2, &boottimebin);
2008		bt = &bt2;
2009	}
2010	switch (BPF_T_FORMAT(tstype)) {
2011	case BPF_T_MICROTIME:
2012		bintime2timeval(bt, &tsm);
2013		ts->bt_sec = tsm.tv_sec;
2014		ts->bt_frac = tsm.tv_usec;
2015		break;
2016	case BPF_T_NANOTIME:
2017		bintime2timespec(bt, &tsn);
2018		ts->bt_sec = tsn.tv_sec;
2019		ts->bt_frac = tsn.tv_nsec;
2020		break;
2021	case BPF_T_BINTIME:
2022		ts->bt_sec = bt->sec;
2023		ts->bt_frac = bt->frac;
2024		break;
2025	}
2026}
2027
2028/*
2029 * Move the packet data from interface memory (pkt) into the
2030 * store buffer.  "cpfn" is the routine called to do the actual data
2031 * transfer.  bcopy is passed in to copy contiguous chunks, while
2032 * bpf_append_mbuf is passed in to copy mbuf chains.  In the latter case,
2033 * pkt is really an mbuf.
2034 */
2035static void
2036catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
2037    void (*cpfn)(struct bpf_d *, caddr_t, u_int, void *, u_int),
2038    struct bintime *bt)
2039{
2040	struct bpf_xhdr hdr;
2041#ifndef BURN_BRIDGES
2042	struct bpf_hdr hdr_old;
2043#ifdef COMPAT_FREEBSD32
2044	struct bpf_hdr32 hdr32_old;
2045#endif
2046#endif
2047	int caplen, curlen, hdrlen, totlen;
2048	int do_wakeup = 0;
2049	int do_timestamp;
2050	int tstype;
2051
2052	BPFD_LOCK_ASSERT(d);
2053
2054	/*
2055	 * Detect whether user space has released a buffer back to us, and if
2056	 * so, move it from being a hold buffer to a free buffer.  This may
2057	 * not be the best place to do it (for example, we might only want to
2058	 * run this check if we need the space), but for now it's a reliable
2059	 * spot to do it.
2060	 */
2061	if (d->bd_fbuf == NULL && bpf_canfreebuf(d)) {
2062		d->bd_fbuf = d->bd_hbuf;
2063		d->bd_hbuf = NULL;
2064		d->bd_hlen = 0;
2065		bpf_buf_reclaimed(d);
2066	}
2067
2068	/*
2069	 * Figure out how many bytes to move.  If the packet is
2070	 * greater or equal to the snapshot length, transfer that
2071	 * much.  Otherwise, transfer the whole packet (unless
2072	 * we hit the buffer size limit).
2073	 */
2074	hdrlen = bpf_hdrlen(d);
2075	totlen = hdrlen + min(snaplen, pktlen);
2076	if (totlen > d->bd_bufsize)
2077		totlen = d->bd_bufsize;
2078
2079	/*
2080	 * Round up the end of the previous packet to the next longword.
2081	 *
2082	 * Drop the packet if there's no room and no hope of room
2083	 * If the packet would overflow the storage buffer or the storage
2084	 * buffer is considered immutable by the buffer model, try to rotate
2085	 * the buffer and wakeup pending processes.
2086	 */
2087#ifdef COMPAT_FREEBSD32
2088	if (d->bd_compat32)
2089		curlen = BPF_WORDALIGN32(d->bd_slen);
2090	else
2091#endif
2092		curlen = BPF_WORDALIGN(d->bd_slen);
2093	if (curlen + totlen > d->bd_bufsize || !bpf_canwritebuf(d)) {
2094		if (d->bd_fbuf == NULL) {
2095			/*
2096			 * There's no room in the store buffer, and no
2097			 * prospect of room, so drop the packet.  Notify the
2098			 * buffer model.
2099			 */
2100			bpf_buffull(d);
2101			++d->bd_dcount;
2102			return;
2103		}
2104		ROTATE_BUFFERS(d);
2105		do_wakeup = 1;
2106		curlen = 0;
2107	} else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT)
2108		/*
2109		 * Immediate mode is set, or the read timeout has already
2110		 * expired during a select call.  A packet arrived, so the
2111		 * reader should be woken up.
2112		 */
2113		do_wakeup = 1;
2114	caplen = totlen - hdrlen;
2115	tstype = d->bd_tstamp;
2116	do_timestamp = tstype != BPF_T_NONE;
2117#ifndef BURN_BRIDGES
2118	if (tstype == BPF_T_NONE || BPF_T_FORMAT(tstype) == BPF_T_MICROTIME) {
2119		struct bpf_ts ts;
2120		if (do_timestamp)
2121			bpf_bintime2ts(bt, &ts, tstype);
2122#ifdef COMPAT_FREEBSD32
2123		if (d->bd_compat32) {
2124			bzero(&hdr32_old, sizeof(hdr32_old));
2125			if (do_timestamp) {
2126				hdr32_old.bh_tstamp.tv_sec = ts.bt_sec;
2127				hdr32_old.bh_tstamp.tv_usec = ts.bt_frac;
2128			}
2129			hdr32_old.bh_datalen = pktlen;
2130			hdr32_old.bh_hdrlen = hdrlen;
2131			hdr32_old.bh_caplen = caplen;
2132			bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr32_old,
2133			    sizeof(hdr32_old));
2134			goto copy;
2135		}
2136#endif
2137		bzero(&hdr_old, sizeof(hdr_old));
2138		if (do_timestamp) {
2139			hdr_old.bh_tstamp.tv_sec = ts.bt_sec;
2140			hdr_old.bh_tstamp.tv_usec = ts.bt_frac;
2141		}
2142		hdr_old.bh_datalen = pktlen;
2143		hdr_old.bh_hdrlen = hdrlen;
2144		hdr_old.bh_caplen = caplen;
2145		bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr_old,
2146		    sizeof(hdr_old));
2147		goto copy;
2148	}
2149#endif
2150
2151	/*
2152	 * Append the bpf header.  Note we append the actual header size, but
2153	 * move forward the length of the header plus padding.
2154	 */
2155	bzero(&hdr, sizeof(hdr));
2156	if (do_timestamp)
2157		bpf_bintime2ts(bt, &hdr.bh_tstamp, tstype);
2158	hdr.bh_datalen = pktlen;
2159	hdr.bh_hdrlen = hdrlen;
2160	hdr.bh_caplen = caplen;
2161	bpf_append_bytes(d, d->bd_sbuf, curlen, &hdr, sizeof(hdr));
2162
2163	/*
2164	 * Copy the packet data into the store buffer and update its length.
2165	 */
2166#ifndef BURN_BRIDGES
2167copy:
2168#endif
2169	(*cpfn)(d, d->bd_sbuf, curlen + hdrlen, pkt, caplen);
2170	d->bd_slen = curlen + totlen;
2171
2172	if (do_wakeup)
2173		bpf_wakeup(d);
2174}
2175
2176/*
2177 * Free buffers currently in use by a descriptor.
2178 * Called on close.
2179 */
2180static void
2181bpf_freed(struct bpf_d *d)
2182{
2183
2184	/*
2185	 * We don't need to lock out interrupts since this descriptor has
2186	 * been detached from its interface and it yet hasn't been marked
2187	 * free.
2188	 */
2189	bpf_free(d);
2190	if (d->bd_rfilter != NULL) {
2191		free((caddr_t)d->bd_rfilter, M_BPF);
2192#ifdef BPF_JITTER
2193		if (d->bd_bfilter != NULL)
2194			bpf_destroy_jit_filter(d->bd_bfilter);
2195#endif
2196	}
2197	if (d->bd_wfilter != NULL)
2198		free((caddr_t)d->bd_wfilter, M_BPF);
2199	mtx_destroy(&d->bd_mtx);
2200}
2201
2202/*
2203 * Attach an interface to bpf.  dlt is the link layer type; hdrlen is the
2204 * fixed size of the link header (variable length headers not yet supported).
2205 */
2206void
2207bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2208{
2209
2210	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
2211}
2212
2213/*
2214 * Attach an interface to bpf.  ifp is a pointer to the structure
2215 * defining the interface to be attached, dlt is the link layer type,
2216 * and hdrlen is the fixed size of the link header (variable length
2217 * headers are not yet supporrted).
2218 */
2219void
2220bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
2221{
2222	struct bpf_if *bp;
2223
2224	bp = malloc(sizeof(*bp), M_BPF, M_NOWAIT | M_ZERO);
2225	if (bp == NULL)
2226		panic("bpfattach");
2227
2228	LIST_INIT(&bp->bif_dlist);
2229	bp->bif_ifp = ifp;
2230	bp->bif_dlt = dlt;
2231	mtx_init(&bp->bif_mtx, "bpf interface lock", NULL, MTX_DEF);
2232	KASSERT(*driverp == NULL, ("bpfattach2: driverp already initialized"));
2233	*driverp = bp;
2234
2235	mtx_lock(&bpf_mtx);
2236	LIST_INSERT_HEAD(&bpf_iflist, bp, bif_next);
2237	mtx_unlock(&bpf_mtx);
2238
2239	bp->bif_hdrlen = hdrlen;
2240
2241	if (bootverbose)
2242		if_printf(ifp, "bpf attached\n");
2243}
2244
2245/*
2246 * Detach bpf from an interface.  This involves detaching each descriptor
2247 * associated with the interface, and leaving bd_bif NULL.  Notify each
2248 * descriptor as it's detached so that any sleepers wake up and get
2249 * ENXIO.
2250 */
2251void
2252bpfdetach(struct ifnet *ifp)
2253{
2254	struct bpf_if	*bp;
2255	struct bpf_d	*d;
2256
2257	/* Locate BPF interface information */
2258	mtx_lock(&bpf_mtx);
2259	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2260		if (ifp == bp->bif_ifp)
2261			break;
2262	}
2263
2264	/* Interface wasn't attached */
2265	if ((bp == NULL) || (bp->bif_ifp == NULL)) {
2266		mtx_unlock(&bpf_mtx);
2267		printf("bpfdetach: %s was not attached\n", ifp->if_xname);
2268		return;
2269	}
2270
2271	LIST_REMOVE(bp, bif_next);
2272	mtx_unlock(&bpf_mtx);
2273
2274	while ((d = LIST_FIRST(&bp->bif_dlist)) != NULL) {
2275		bpf_detachd(d);
2276		BPFD_LOCK(d);
2277		bpf_wakeup(d);
2278		BPFD_UNLOCK(d);
2279	}
2280
2281	mtx_destroy(&bp->bif_mtx);
2282	free(bp, M_BPF);
2283}
2284
2285/*
2286 * Get a list of available data link type of the interface.
2287 */
2288static int
2289bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl)
2290{
2291	int n, error;
2292	struct ifnet *ifp;
2293	struct bpf_if *bp;
2294
2295	ifp = d->bd_bif->bif_ifp;
2296	n = 0;
2297	error = 0;
2298	mtx_lock(&bpf_mtx);
2299	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2300		if (bp->bif_ifp != ifp)
2301			continue;
2302		if (bfl->bfl_list != NULL) {
2303			if (n >= bfl->bfl_len) {
2304				mtx_unlock(&bpf_mtx);
2305				return (ENOMEM);
2306			}
2307			error = copyout(&bp->bif_dlt,
2308			    bfl->bfl_list + n, sizeof(u_int));
2309		}
2310		n++;
2311	}
2312	mtx_unlock(&bpf_mtx);
2313	bfl->bfl_len = n;
2314	return (error);
2315}
2316
2317/*
2318 * Set the data link type of a BPF instance.
2319 */
2320static int
2321bpf_setdlt(struct bpf_d *d, u_int dlt)
2322{
2323	int error, opromisc;
2324	struct ifnet *ifp;
2325	struct bpf_if *bp;
2326
2327	if (d->bd_bif->bif_dlt == dlt)
2328		return (0);
2329	ifp = d->bd_bif->bif_ifp;
2330	mtx_lock(&bpf_mtx);
2331	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2332		if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
2333			break;
2334	}
2335	mtx_unlock(&bpf_mtx);
2336	if (bp != NULL) {
2337		opromisc = d->bd_promisc;
2338		bpf_detachd(d);
2339		bpf_attachd(d, bp);
2340		BPFD_LOCK(d);
2341		reset_d(d);
2342		BPFD_UNLOCK(d);
2343		if (opromisc) {
2344			error = ifpromisc(bp->bif_ifp, 1);
2345			if (error)
2346				if_printf(bp->bif_ifp,
2347					"bpf_setdlt: ifpromisc failed (%d)\n",
2348					error);
2349			else
2350				d->bd_promisc = 1;
2351		}
2352	}
2353	return (bp == NULL ? EINVAL : 0);
2354}
2355
2356static void
2357bpf_drvinit(void *unused)
2358{
2359	struct cdev *dev;
2360
2361	mtx_init(&bpf_mtx, "bpf global lock", NULL, MTX_DEF);
2362	LIST_INIT(&bpf_iflist);
2363
2364	dev = make_dev(&bpf_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "bpf");
2365	/* For compatibility */
2366	make_dev_alias(dev, "bpf0");
2367}
2368
2369/*
2370 * Zero out the various packet counters associated with all of the bpf
2371 * descriptors.  At some point, we will probably want to get a bit more
2372 * granular and allow the user to specify descriptors to be zeroed.
2373 */
2374static void
2375bpf_zero_counters(void)
2376{
2377	struct bpf_if *bp;
2378	struct bpf_d *bd;
2379
2380	mtx_lock(&bpf_mtx);
2381	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2382		BPFIF_LOCK(bp);
2383		LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
2384			BPFD_LOCK(bd);
2385			bd->bd_rcount = 0;
2386			bd->bd_dcount = 0;
2387			bd->bd_fcount = 0;
2388			bd->bd_wcount = 0;
2389			bd->bd_wfcount = 0;
2390			bd->bd_zcopy = 0;
2391			BPFD_UNLOCK(bd);
2392		}
2393		BPFIF_UNLOCK(bp);
2394	}
2395	mtx_unlock(&bpf_mtx);
2396}
2397
2398static void
2399bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd)
2400{
2401
2402	bzero(d, sizeof(*d));
2403	BPFD_LOCK_ASSERT(bd);
2404	d->bd_structsize = sizeof(*d);
2405	d->bd_immediate = bd->bd_immediate;
2406	d->bd_promisc = bd->bd_promisc;
2407	d->bd_hdrcmplt = bd->bd_hdrcmplt;
2408	d->bd_direction = bd->bd_direction;
2409	d->bd_feedback = bd->bd_feedback;
2410	d->bd_async = bd->bd_async;
2411	d->bd_rcount = bd->bd_rcount;
2412	d->bd_dcount = bd->bd_dcount;
2413	d->bd_fcount = bd->bd_fcount;
2414	d->bd_sig = bd->bd_sig;
2415	d->bd_slen = bd->bd_slen;
2416	d->bd_hlen = bd->bd_hlen;
2417	d->bd_bufsize = bd->bd_bufsize;
2418	d->bd_pid = bd->bd_pid;
2419	strlcpy(d->bd_ifname,
2420	    bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ);
2421	d->bd_locked = bd->bd_locked;
2422	d->bd_wcount = bd->bd_wcount;
2423	d->bd_wdcount = bd->bd_wdcount;
2424	d->bd_wfcount = bd->bd_wfcount;
2425	d->bd_zcopy = bd->bd_zcopy;
2426	d->bd_bufmode = bd->bd_bufmode;
2427}
2428
2429static int
2430bpf_stats_sysctl(SYSCTL_HANDLER_ARGS)
2431{
2432	struct xbpf_d *xbdbuf, *xbd, zerostats;
2433	int index, error;
2434	struct bpf_if *bp;
2435	struct bpf_d *bd;
2436
2437	/*
2438	 * XXX This is not technically correct. It is possible for non
2439	 * privileged users to open bpf devices. It would make sense
2440	 * if the users who opened the devices were able to retrieve
2441	 * the statistics for them, too.
2442	 */
2443	error = priv_check(req->td, PRIV_NET_BPF);
2444	if (error)
2445		return (error);
2446	/*
2447	 * Check to see if the user is requesting that the counters be
2448	 * zeroed out.  Explicitly check that the supplied data is zeroed,
2449	 * as we aren't allowing the user to set the counters currently.
2450	 */
2451	if (req->newptr != NULL) {
2452		if (req->newlen != sizeof(zerostats))
2453			return (EINVAL);
2454		bzero(&zerostats, sizeof(zerostats));
2455		xbd = req->newptr;
2456		if (bcmp(xbd, &zerostats, sizeof(*xbd)) != 0)
2457			return (EINVAL);
2458		bpf_zero_counters();
2459		return (0);
2460	}
2461	if (req->oldptr == NULL)
2462		return (SYSCTL_OUT(req, 0, bpf_bpfd_cnt * sizeof(*xbd)));
2463	if (bpf_bpfd_cnt == 0)
2464		return (SYSCTL_OUT(req, 0, 0));
2465	xbdbuf = malloc(req->oldlen, M_BPF, M_WAITOK);
2466	mtx_lock(&bpf_mtx);
2467	if (req->oldlen < (bpf_bpfd_cnt * sizeof(*xbd))) {
2468		mtx_unlock(&bpf_mtx);
2469		free(xbdbuf, M_BPF);
2470		return (ENOMEM);
2471	}
2472	index = 0;
2473	LIST_FOREACH(bp, &bpf_iflist, bif_next) {
2474		BPFIF_LOCK(bp);
2475		LIST_FOREACH(bd, &bp->bif_dlist, bd_next) {
2476			xbd = &xbdbuf[index++];
2477			BPFD_LOCK(bd);
2478			bpfstats_fill_xbpf(xbd, bd);
2479			BPFD_UNLOCK(bd);
2480		}
2481		BPFIF_UNLOCK(bp);
2482	}
2483	mtx_unlock(&bpf_mtx);
2484	error = SYSCTL_OUT(req, xbdbuf, index * sizeof(*xbd));
2485	free(xbdbuf, M_BPF);
2486	return (error);
2487}
2488
2489SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE,bpf_drvinit,NULL);
2490
2491#else /* !DEV_BPF && !NETGRAPH_BPF */
2492/*
2493 * NOP stubs to allow bpf-using drivers to load and function.
2494 *
2495 * A 'better' implementation would allow the core bpf functionality
2496 * to be loaded at runtime.
2497 */
2498static struct bpf_if bp_null;
2499
2500void
2501bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen)
2502{
2503}
2504
2505void
2506bpf_mtap(struct bpf_if *bp, struct mbuf *m)
2507{
2508}
2509
2510void
2511bpf_mtap2(struct bpf_if *bp, void *d, u_int l, struct mbuf *m)
2512{
2513}
2514
2515void
2516bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
2517{
2518
2519	bpfattach2(ifp, dlt, hdrlen, &ifp->if_bpf);
2520}
2521
2522void
2523bpfattach2(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp)
2524{
2525
2526	*driverp = &bp_null;
2527}
2528
2529void
2530bpfdetach(struct ifnet *ifp)
2531{
2532}
2533
2534u_int
2535bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen)
2536{
2537	return -1;	/* "no filter" behaviour */
2538}
2539
2540int
2541bpf_validate(const struct bpf_insn *f, int len)
2542{
2543	return 0;		/* false */
2544}
2545
2546#endif /* !DEV_BPF && !NETGRAPH_BPF */
2547