1/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
2#ifndef __BPF_HELPERS__
3#define __BPF_HELPERS__
4
5/*
6 * Note that bpf programs need to include either
7 * vmlinux.h (auto-generated from BTF) or linux/types.h
8 * in advance since bpf_helper_defs.h uses such types
9 * as __u64.
10 */
11#include "bpf_helper_defs.h"
12
13#define __uint(name, val) int (*name)[val]
14#define __type(name, val) typeof(val) *name
15#define __array(name, val) typeof(val) *name[]
16
17/*
18 * Helper macro to place programs, maps, license in
19 * different sections in elf_bpf file. Section names
20 * are interpreted by libbpf depending on the context (BPF programs, BPF maps,
21 * extern variables, etc).
22 * To allow use of SEC() with externs (e.g., for extern .maps declarations),
23 * make sure __attribute__((unused)) doesn't trigger compilation warning.
24 */
25#if __GNUC__ && !__clang__
26
27/*
28 * Pragma macros are broken on GCC
29 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55578
30 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90400
31 */
32#define SEC(name) __attribute__((section(name), used))
33
34#else
35
36#define SEC(name) \
37	_Pragma("GCC diagnostic push")					    \
38	_Pragma("GCC diagnostic ignored \"-Wignored-attributes\"")	    \
39	__attribute__((section(name), used))				    \
40	_Pragma("GCC diagnostic pop")					    \
41
42#endif
43
44/* Avoid 'linux/stddef.h' definition of '__always_inline'. */
45#undef __always_inline
46#define __always_inline inline __attribute__((always_inline))
47
48#ifndef __noinline
49#define __noinline __attribute__((noinline))
50#endif
51#ifndef __weak
52#define __weak __attribute__((weak))
53#endif
54
55/*
56 * Use __hidden attribute to mark a non-static BPF subprogram effectively
57 * static for BPF verifier's verification algorithm purposes, allowing more
58 * extensive and permissive BPF verification process, taking into account
59 * subprogram's caller context.
60 */
61#define __hidden __attribute__((visibility("hidden")))
62
63/* When utilizing vmlinux.h with BPF CO-RE, user BPF programs can't include
64 * any system-level headers (such as stddef.h, linux/version.h, etc), and
65 * commonly-used macros like NULL and KERNEL_VERSION aren't available through
66 * vmlinux.h. This just adds unnecessary hurdles and forces users to re-define
67 * them on their own. So as a convenience, provide such definitions here.
68 */
69#ifndef NULL
70#define NULL ((void *)0)
71#endif
72
73#ifndef KERNEL_VERSION
74#define KERNEL_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
75#endif
76
77/*
78 * Helper macros to manipulate data structures
79 */
80#ifndef offsetof
81#define offsetof(TYPE, MEMBER)	((unsigned long)&((TYPE *)0)->MEMBER)
82#endif
83#ifndef container_of
84#define container_of(ptr, type, member)				\
85	({							\
86		void *__mptr = (void *)(ptr);			\
87		((type *)(__mptr - offsetof(type, member)));	\
88	})
89#endif
90
91/*
92 * Compiler (optimization) barrier.
93 */
94#ifndef barrier
95#define barrier() asm volatile("" ::: "memory")
96#endif
97
98/* Variable-specific compiler (optimization) barrier. It's a no-op which makes
99 * compiler believe that there is some black box modification of a given
100 * variable and thus prevents compiler from making extra assumption about its
101 * value and potential simplifications and optimizations on this variable.
102 *
103 * E.g., compiler might often delay or even omit 32-bit to 64-bit casting of
104 * a variable, making some code patterns unverifiable. Putting barrier_var()
105 * in place will ensure that cast is performed before the barrier_var()
106 * invocation, because compiler has to pessimistically assume that embedded
107 * asm section might perform some extra operations on that variable.
108 *
109 * This is a variable-specific variant of more global barrier().
110 */
111#ifndef barrier_var
112#define barrier_var(var) asm volatile("" : "=r"(var) : "0"(var))
113#endif
114
115/*
116 * Helper macro to throw a compilation error if __bpf_unreachable() gets
117 * built into the resulting code. This works given BPF back end does not
118 * implement __builtin_trap(). This is useful to assert that certain paths
119 * of the program code are never used and hence eliminated by the compiler.
120 *
121 * For example, consider a switch statement that covers known cases used by
122 * the program. __bpf_unreachable() can then reside in the default case. If
123 * the program gets extended such that a case is not covered in the switch
124 * statement, then it will throw a build error due to the default case not
125 * being compiled out.
126 */
127#ifndef __bpf_unreachable
128# define __bpf_unreachable()	__builtin_trap()
129#endif
130
131/*
132 * Helper function to perform a tail call with a constant/immediate map slot.
133 */
134#if __clang_major__ >= 8 && defined(__bpf__)
135static __always_inline void
136bpf_tail_call_static(void *ctx, const void *map, const __u32 slot)
137{
138	if (!__builtin_constant_p(slot))
139		__bpf_unreachable();
140
141	/*
142	 * Provide a hard guarantee that LLVM won't optimize setting r2 (map
143	 * pointer) and r3 (constant map index) from _different paths_ ending
144	 * up at the _same_ call insn as otherwise we won't be able to use the
145	 * jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel
146	 * given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key
147	 * tracking for prog array pokes") for details on verifier tracking.
148	 *
149	 * Note on clobber list: we need to stay in-line with BPF calling
150	 * convention, so even if we don't end up using r0, r4, r5, we need
151	 * to mark them as clobber so that LLVM doesn't end up using them
152	 * before / after the call.
153	 */
154	asm volatile("r1 = %[ctx]\n\t"
155		     "r2 = %[map]\n\t"
156		     "r3 = %[slot]\n\t"
157		     "call 12"
158		     :: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot)
159		     : "r0", "r1", "r2", "r3", "r4", "r5");
160}
161#endif
162
163/*
164 * Helper structure used by eBPF C program
165 * to describe BPF map attributes to libbpf loader
166 */
167struct bpf_map_def {
168	unsigned int type;
169	unsigned int key_size;
170	unsigned int value_size;
171	unsigned int max_entries;
172	unsigned int map_flags;
173} __attribute__((deprecated("use BTF-defined maps in .maps section")));
174
175enum libbpf_pin_type {
176	LIBBPF_PIN_NONE,
177	/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
178	LIBBPF_PIN_BY_NAME,
179};
180
181enum libbpf_tristate {
182	TRI_NO = 0,
183	TRI_YES = 1,
184	TRI_MODULE = 2,
185};
186
187#define __kconfig __attribute__((section(".kconfig")))
188#define __ksym __attribute__((section(".ksyms")))
189#define __kptr __attribute__((btf_type_tag("kptr")))
190#define __kptr_ref __attribute__((btf_type_tag("kptr_ref")))
191
192#ifndef ___bpf_concat
193#define ___bpf_concat(a, b) a ## b
194#endif
195#ifndef ___bpf_apply
196#define ___bpf_apply(fn, n) ___bpf_concat(fn, n)
197#endif
198#ifndef ___bpf_nth
199#define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N
200#endif
201#ifndef ___bpf_narg
202#define ___bpf_narg(...) \
203	___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
204#endif
205
206#define ___bpf_fill0(arr, p, x) do {} while (0)
207#define ___bpf_fill1(arr, p, x) arr[p] = x
208#define ___bpf_fill2(arr, p, x, args...) arr[p] = x; ___bpf_fill1(arr, p + 1, args)
209#define ___bpf_fill3(arr, p, x, args...) arr[p] = x; ___bpf_fill2(arr, p + 1, args)
210#define ___bpf_fill4(arr, p, x, args...) arr[p] = x; ___bpf_fill3(arr, p + 1, args)
211#define ___bpf_fill5(arr, p, x, args...) arr[p] = x; ___bpf_fill4(arr, p + 1, args)
212#define ___bpf_fill6(arr, p, x, args...) arr[p] = x; ___bpf_fill5(arr, p + 1, args)
213#define ___bpf_fill7(arr, p, x, args...) arr[p] = x; ___bpf_fill6(arr, p + 1, args)
214#define ___bpf_fill8(arr, p, x, args...) arr[p] = x; ___bpf_fill7(arr, p + 1, args)
215#define ___bpf_fill9(arr, p, x, args...) arr[p] = x; ___bpf_fill8(arr, p + 1, args)
216#define ___bpf_fill10(arr, p, x, args...) arr[p] = x; ___bpf_fill9(arr, p + 1, args)
217#define ___bpf_fill11(arr, p, x, args...) arr[p] = x; ___bpf_fill10(arr, p + 1, args)
218#define ___bpf_fill12(arr, p, x, args...) arr[p] = x; ___bpf_fill11(arr, p + 1, args)
219#define ___bpf_fill(arr, args...) \
220	___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args)
221
222/*
223 * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values
224 * in a structure.
225 */
226#define BPF_SEQ_PRINTF(seq, fmt, args...)			\
227({								\
228	static const char ___fmt[] = fmt;			\
229	unsigned long long ___param[___bpf_narg(args)];		\
230								\
231	_Pragma("GCC diagnostic push")				\
232	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")	\
233	___bpf_fill(___param, args);				\
234	_Pragma("GCC diagnostic pop")				\
235								\
236	bpf_seq_printf(seq, ___fmt, sizeof(___fmt),		\
237		       ___param, sizeof(___param));		\
238})
239
240/*
241 * BPF_SNPRINTF wraps the bpf_snprintf helper with variadic arguments instead of
242 * an array of u64.
243 */
244#define BPF_SNPRINTF(out, out_size, fmt, args...)		\
245({								\
246	static const char ___fmt[] = fmt;			\
247	unsigned long long ___param[___bpf_narg(args)];		\
248								\
249	_Pragma("GCC diagnostic push")				\
250	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")	\
251	___bpf_fill(___param, args);				\
252	_Pragma("GCC diagnostic pop")				\
253								\
254	bpf_snprintf(out, out_size, ___fmt,			\
255		     ___param, sizeof(___param));		\
256})
257
258#ifdef BPF_NO_GLOBAL_DATA
259#define BPF_PRINTK_FMT_MOD
260#else
261#define BPF_PRINTK_FMT_MOD static const
262#endif
263
264#define __bpf_printk(fmt, ...)				\
265({							\
266	BPF_PRINTK_FMT_MOD char ____fmt[] = fmt;	\
267	bpf_trace_printk(____fmt, sizeof(____fmt),	\
268			 ##__VA_ARGS__);		\
269})
270
271/*
272 * __bpf_vprintk wraps the bpf_trace_vprintk helper with variadic arguments
273 * instead of an array of u64.
274 */
275#define __bpf_vprintk(fmt, args...)				\
276({								\
277	static const char ___fmt[] = fmt;			\
278	unsigned long long ___param[___bpf_narg(args)];		\
279								\
280	_Pragma("GCC diagnostic push")				\
281	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")	\
282	___bpf_fill(___param, args);				\
283	_Pragma("GCC diagnostic pop")				\
284								\
285	bpf_trace_vprintk(___fmt, sizeof(___fmt),		\
286			  ___param, sizeof(___param));		\
287})
288
289/* Use __bpf_printk when bpf_printk call has 3 or fewer fmt args
290 * Otherwise use __bpf_vprintk
291 */
292#define ___bpf_pick_printk(...) \
293	___bpf_nth(_, ##__VA_ARGS__, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk,	\
294		   __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk,		\
295		   __bpf_vprintk, __bpf_vprintk, __bpf_printk /*3*/, __bpf_printk /*2*/,\
296		   __bpf_printk /*1*/, __bpf_printk /*0*/)
297
298/* Helper macro to print out debug messages */
299#define bpf_printk(fmt, args...) ___bpf_pick_printk(args)(fmt, ##args)
300
301#endif
302