1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __PACKET_INTERNAL_H__
3#define __PACKET_INTERNAL_H__
4
5#include <linux/refcount.h>
6
7struct packet_mclist {
8	struct packet_mclist	*next;
9	int			ifindex;
10	int			count;
11	unsigned short		type;
12	unsigned short		alen;
13	unsigned char		addr[MAX_ADDR_LEN];
14};
15
16/* kbdq - kernel block descriptor queue */
17struct tpacket_kbdq_core {
18	struct pgv	*pkbdq;
19	unsigned int	feature_req_word;
20	unsigned int	hdrlen;
21	unsigned char	reset_pending_on_curr_blk;
22	unsigned char   delete_blk_timer;
23	unsigned short	kactive_blk_num;
24	unsigned short	blk_sizeof_priv;
25
26	/* last_kactive_blk_num:
27	 * trick to see if user-space has caught up
28	 * in order to avoid refreshing timer when every single pkt arrives.
29	 */
30	unsigned short	last_kactive_blk_num;
31
32	char		*pkblk_start;
33	char		*pkblk_end;
34	int		kblk_size;
35	unsigned int	max_frame_len;
36	unsigned int	knum_blocks;
37	uint64_t	knxt_seq_num;
38	char		*prev;
39	char		*nxt_offset;
40	struct sk_buff	*skb;
41
42	rwlock_t	blk_fill_in_prog_lock;
43
44	/* Default is set to 8ms */
45#define DEFAULT_PRB_RETIRE_TOV	(8)
46
47	unsigned short  retire_blk_tov;
48	unsigned short  version;
49	unsigned long	tov_in_jiffies;
50
51	/* timer to retire an outstanding block */
52	struct timer_list retire_blk_timer;
53};
54
55struct pgv {
56	char *buffer;
57};
58
59struct packet_ring_buffer {
60	struct pgv		*pg_vec;
61
62	unsigned int		head;
63	unsigned int		frames_per_block;
64	unsigned int		frame_size;
65	unsigned int		frame_max;
66
67	unsigned int		pg_vec_order;
68	unsigned int		pg_vec_pages;
69	unsigned int		pg_vec_len;
70
71	unsigned int __percpu	*pending_refcnt;
72
73	union {
74		unsigned long			*rx_owner_map;
75		struct tpacket_kbdq_core	prb_bdqc;
76	};
77};
78
79extern struct mutex fanout_mutex;
80#define PACKET_FANOUT_MAX	(1 << 16)
81
82struct packet_fanout {
83	possible_net_t		net;
84	unsigned int		num_members;
85	u32			max_num_members;
86	u16			id;
87	u8			type;
88	u8			flags;
89	union {
90		atomic_t		rr_cur;
91		struct bpf_prog __rcu	*bpf_prog;
92	};
93	struct list_head	list;
94	spinlock_t		lock;
95	refcount_t		sk_ref;
96	struct packet_type	prot_hook ____cacheline_aligned_in_smp;
97	struct sock	__rcu	*arr[] __counted_by(max_num_members);
98};
99
100struct packet_rollover {
101	int			sock;
102	atomic_long_t		num;
103	atomic_long_t		num_huge;
104	atomic_long_t		num_failed;
105#define ROLLOVER_HLEN	(L1_CACHE_BYTES / sizeof(u32))
106	u32			history[ROLLOVER_HLEN] ____cacheline_aligned;
107} ____cacheline_aligned_in_smp;
108
109struct packet_sock {
110	/* struct sock has to be the first member of packet_sock */
111	struct sock		sk;
112	struct packet_fanout	*fanout;
113	union  tpacket_stats_u	stats;
114	struct packet_ring_buffer	rx_ring;
115	struct packet_ring_buffer	tx_ring;
116	int			copy_thresh;
117	spinlock_t		bind_lock;
118	struct mutex		pg_vec_lock;
119	unsigned long		flags;
120	int			ifindex;	/* bound device		*/
121	u8			vnet_hdr_sz;
122	__be16			num;
123	struct packet_rollover	*rollover;
124	struct packet_mclist	*mclist;
125	atomic_long_t		mapped;
126	enum tpacket_versions	tp_version;
127	unsigned int		tp_hdrlen;
128	unsigned int		tp_reserve;
129	unsigned int		tp_tstamp;
130	struct completion	skb_completion;
131	struct net_device __rcu	*cached_dev;
132	struct packet_type	prot_hook ____cacheline_aligned_in_smp;
133	atomic_t		tp_drops ____cacheline_aligned_in_smp;
134};
135
136#define pkt_sk(ptr) container_of_const(ptr, struct packet_sock, sk)
137
138enum packet_sock_flags {
139	PACKET_SOCK_ORIGDEV,
140	PACKET_SOCK_AUXDATA,
141	PACKET_SOCK_TX_HAS_OFF,
142	PACKET_SOCK_TP_LOSS,
143	PACKET_SOCK_RUNNING,
144	PACKET_SOCK_PRESSURE,
145	PACKET_SOCK_QDISC_BYPASS,
146};
147
148static inline void packet_sock_flag_set(struct packet_sock *po,
149					enum packet_sock_flags flag,
150					bool val)
151{
152	if (val)
153		set_bit(flag, &po->flags);
154	else
155		clear_bit(flag, &po->flags);
156}
157
158static inline bool packet_sock_flag(const struct packet_sock *po,
159				    enum packet_sock_flags flag)
160{
161	return test_bit(flag, &po->flags);
162}
163
164#endif
165