bpf_jit_machdep.h revision 331722 
1/*-
2 * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
3 * Copyright (C) 2005-2009 Jung-uk Kim <jkim@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the Politecnico di Torino nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * $FreeBSD: stable/11/sys/amd64/amd64/bpf_jit_machdep.h 331722 2018-03-29 02:50:57Z eadler $
32 */
33
34#ifndef _BPF_JIT_MACHDEP_H_
35#define _BPF_JIT_MACHDEP_H_
36
37/*
38 * Registers
39 */
40#define RAX	0
41#define RCX	1
42#define RDX	2
43#define RBX	3
44#define RSP	4
45#define RBP	5
46#define RSI	6
47#define RDI	7
48#define R8	0
49#define R9	1
50#define R10	2
51#define R11	3
52#define R12	4
53#define R13	5
54#define R14	6
55#define R15	7
56
57#define EAX	0
58#define ECX	1
59#define EDX	2
60#define EBX	3
61#define ESP	4
62#define EBP	5
63#define ESI	6
64#define EDI	7
65#define R8D	0
66#define R9D	1
67#define R10D	2
68#define R11D	3
69#define R12D	4
70#define R13D	5
71#define R14D	6
72#define R15D	7
73
74#define AX	0
75#define CX	1
76#define DX	2
77#define BX	3
78#define SP	4
79#define BP	5
80#define SI	6
81#define DI	7
82
83#define AL	0
84#define CL	1
85#define DL	2
86#define BL	3
87
88/* Optimization flags */
89#define	BPF_JIT_FRET	0x01
90#define	BPF_JIT_FPKT	0x02
91#define	BPF_JIT_FMEM	0x04
92#define	BPF_JIT_FJMP	0x08
93#define	BPF_JIT_FLEN	0x10
94
95#define	BPF_JIT_FLAG_ALL	\
96    (BPF_JIT_FPKT | BPF_JIT_FMEM | BPF_JIT_FJMP | BPF_JIT_FLEN)
97
98/* A stream of native binary code */
99typedef struct bpf_bin_stream {
100	/* Current native instruction pointer. */
101	int		cur_ip;
102
103	/*
104	 * Current BPF instruction pointer, i.e. position in
105	 * the BPF program reached by the jitter.
106	 */
107	int		bpf_pc;
108
109	/* Instruction buffer, contains the generated native code. */
110	char		*ibuf;
111
112	/* Jumps reference table. */
113	u_int		*refs;
114} bpf_bin_stream;
115
116/*
117 * Prototype of the emit functions.
118 *
119 * Different emit functions are used to create the reference table and
120 * to generate the actual filtering code. This allows to have simpler
121 * instruction macros.
122 * The first parameter is the stream that will receive the data.
123 * The second one is a variable containing the data.
124 * The third one is the length, that can be 1, 2, or 4 since it is possible
125 * to emit a byte, a short, or a word at a time.
126 */
127typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
128
129/*
130 * Native instruction macros
131 */
132
133/* movl i32,r32 */
134#define MOVid(i32, r32) do {						\
135	emitm(&stream, (11 << 4) | (1 << 3) | (r32 & 0x7), 1);		\
136	emitm(&stream, i32, 4);						\
137} while (0)
138
139/* movq i64,r64 */
140#define MOViq(i64, r64) do {						\
141	emitm(&stream, 0x48, 1);					\
142	emitm(&stream, (11 << 4) | (1 << 3) | (r64 & 0x7), 1);		\
143	emitm(&stream, i64, 4);						\
144	emitm(&stream, (i64 >> 32), 4);					\
145} while (0)
146
147/* movl sr32,dr32 */
148#define MOVrd(sr32, dr32) do {						\
149	emitm(&stream, 0x89, 1);					\
150	emitm(&stream,							\
151	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
152} while (0)
153
154/* movl sr32,dr32 (dr32 = %r8-15d) */
155#define MOVrd2(sr32, dr32) do {						\
156	emitm(&stream, 0x8941, 2);					\
157	emitm(&stream,							\
158	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
159} while (0)
160
161/* movl sr32,dr32 (sr32 = %r8-15d) */
162#define MOVrd3(sr32, dr32) do {						\
163	emitm(&stream, 0x8944, 2);					\
164	emitm(&stream,							\
165	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
166} while (0)
167
168/* movq sr64,dr64 */
169#define MOVrq(sr64, dr64) do {						\
170	emitm(&stream, 0x8948, 2);					\
171	emitm(&stream,							\
172	    (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
173} while (0)
174
175/* movq sr64,dr64 (dr64 = %r8-15) */
176#define MOVrq2(sr64, dr64) do {						\
177	emitm(&stream, 0x8949, 2);					\
178	emitm(&stream,							\
179	    (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
180} while (0)
181
182/* movq sr64,dr64 (sr64 = %r8-15) */
183#define MOVrq3(sr64, dr64) do {						\
184	emitm(&stream, 0x894c, 2);					\
185	emitm(&stream,							\
186	    (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
187} while (0)
188
189/* movl (sr64,or64,1),dr32 */
190#define MOVobd(sr64, or64, dr32) do {					\
191	emitm(&stream, 0x8b, 1);					\
192	emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1);			\
193	emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);		\
194} while (0)
195
196/* movw (sr64,or64,1),dr16 */
197#define MOVobw(sr64, or64, dr16) do {					\
198	emitm(&stream, 0x8b66, 2);					\
199	emitm(&stream, ((dr16 & 0x7) << 3) | 4, 1);			\
200	emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);		\
201} while (0)
202
203/* movb (sr64,or64,1),dr8 */
204#define MOVobb(sr64, or64, dr8) do {					\
205	emitm(&stream, 0x8a, 1);					\
206	emitm(&stream, ((dr8 & 0x7) << 3) | 4, 1);			\
207	emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);		\
208} while (0)
209
210/* movl sr32,(dr64,or64,1) */
211#define MOVomd(sr32, dr64, or64) do {					\
212	emitm(&stream, 0x89, 1);					\
213	emitm(&stream, ((sr32 & 0x7) << 3) | 4, 1);			\
214	emitm(&stream, ((or64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
215} while (0)
216
217/* bswapl dr32 */
218#define BSWAP(dr32) do {						\
219	emitm(&stream, 0xf, 1);						\
220	emitm(&stream, (0x19 << 3) | dr32, 1);				\
221} while (0)
222
223/* xchgb %al,%ah */
224#define SWAP_AX() do {							\
225	emitm(&stream, 0xc486, 2);					\
226} while (0)
227
228/* pushq r64 */
229#define PUSH(r64) do {							\
230	emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1);		\
231} while (0)
232
233/* leaveq */
234#define LEAVE() do {							\
235	emitm(&stream, 0xc9, 1);					\
236} while (0)
237
238/* retq */
239#define RET() do {							\
240	emitm(&stream, 0xc3, 1);					\
241} while (0)
242
243/* addl sr32,dr32 */
244#define ADDrd(sr32, dr32) do {						\
245	emitm(&stream, 0x01, 1);					\
246	emitm(&stream,							\
247	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
248} while (0)
249
250/* addl i32,%eax */
251#define ADD_EAXi(i32) do {						\
252	emitm(&stream, 0x05, 1);					\
253	emitm(&stream, i32, 4);						\
254} while (0)
255
256/* addl i8,r32 */
257#define ADDib(i8, r32) do {						\
258	emitm(&stream, 0x83, 1);					\
259	emitm(&stream, (24 << 3) | r32, 1);				\
260	emitm(&stream, i8, 1);						\
261} while (0)
262
263/* subl sr32,dr32 */
264#define SUBrd(sr32, dr32) do {						\
265	emitm(&stream, 0x29, 1);					\
266	emitm(&stream,							\
267	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
268} while (0)
269
270/* subl i32,%eax */
271#define SUB_EAXi(i32) do {						\
272	emitm(&stream, 0x2d, 1);					\
273	emitm(&stream, i32, 4);						\
274} while (0)
275
276/* subq i8,r64 */
277#define SUBib(i8, r64) do {						\
278	emitm(&stream, 0x8348, 2);					\
279	emitm(&stream, (29 << 3) | (r64 & 0x7), 1);			\
280	emitm(&stream, i8, 1);						\
281} while (0)
282
283/* mull r32 */
284#define MULrd(r32) do {							\
285	emitm(&stream, 0xf7, 1);					\
286	emitm(&stream, (7 << 5) | (r32 & 0x7), 1);			\
287} while (0)
288
289/* divl r32 */
290#define DIVrd(r32) do {							\
291	emitm(&stream, 0xf7, 1);					\
292	emitm(&stream, (15 << 4) | (r32 & 0x7), 1);			\
293} while (0)
294
295/* andb i8,r8 */
296#define ANDib(i8, r8) do {						\
297	if (r8 == AL) {							\
298		emitm(&stream, 0x24, 1);				\
299	} else {							\
300		emitm(&stream, 0x80, 1);				\
301		emitm(&stream, (7 << 5) | r8, 1);			\
302	}								\
303	emitm(&stream, i8, 1);						\
304} while (0)
305
306/* andl i32,r32 */
307#define ANDid(i32, r32) do {						\
308	if (r32 == EAX) {						\
309		emitm(&stream, 0x25, 1);				\
310	} else {							\
311		emitm(&stream, 0x81, 1);				\
312		emitm(&stream, (7 << 5) | r32, 1);			\
313	}								\
314	emitm(&stream, i32, 4);						\
315} while (0)
316
317/* andl sr32,dr32 */
318#define ANDrd(sr32, dr32) do {						\
319	emitm(&stream, 0x21, 1);					\
320	emitm(&stream,							\
321	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
322} while (0)
323
324/* testl i32,r32 */
325#define TESTid(i32, r32) do {						\
326	if (r32 == EAX) {						\
327		emitm(&stream, 0xa9, 1);				\
328	} else {							\
329		emitm(&stream, 0xf7, 1);				\
330		emitm(&stream, (3 << 6) | r32, 1);			\
331	}								\
332	emitm(&stream, i32, 4);						\
333} while (0)
334
335/* testl sr32,dr32 */
336#define TESTrd(sr32, dr32) do {						\
337	emitm(&stream, 0x85, 1);					\
338	emitm(&stream,							\
339	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
340} while (0)
341
342/* orl sr32,dr32 */
343#define ORrd(sr32, dr32) do {						\
344	emitm(&stream, 0x09, 1);					\
345	emitm(&stream,							\
346	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
347} while (0)
348
349/* orl i32,r32 */
350#define ORid(i32, r32) do {						\
351	if (r32 == EAX) {						\
352		emitm(&stream, 0x0d, 1);				\
353	} else {							\
354		emitm(&stream, 0x81, 1);				\
355		emitm(&stream, (25 << 3) | r32, 1);			\
356	}								\
357	emitm(&stream, i32, 4);						\
358} while (0)
359
360/* shll i8,r32 */
361#define SHLib(i8, r32) do {						\
362	emitm(&stream, 0xc1, 1);					\
363	emitm(&stream, (7 << 5) | (r32 & 0x7), 1);			\
364	emitm(&stream, i8, 1);						\
365} while (0)
366
367/* shll %cl,dr32 */
368#define SHL_CLrb(dr32) do {						\
369	emitm(&stream, 0xd3, 1);					\
370	emitm(&stream, (7 << 5) | (dr32 & 0x7), 1);			\
371} while (0)
372
373/* shrl i8,r32 */
374#define SHRib(i8, r32) do {						\
375	emitm(&stream, 0xc1, 1);					\
376	emitm(&stream, (29 << 3) | (r32 & 0x7), 1);			\
377	emitm(&stream, i8, 1);						\
378} while (0)
379
380/* shrl %cl,dr32 */
381#define SHR_CLrb(dr32) do {						\
382	emitm(&stream, 0xd3, 1);					\
383	emitm(&stream, (29 << 3) | (dr32 & 0x7), 1);			\
384} while (0)
385
386/* negl r32 */
387#define NEGd(r32) do {							\
388	emitm(&stream, 0xf7, 1);					\
389	emitm(&stream, (27 << 3) | (r32 & 0x7), 1);			\
390} while (0)
391
392/* cmpl sr32,dr32 */
393#define CMPrd(sr32, dr32) do {						\
394	emitm(&stream, 0x39, 1);					\
395	emitm(&stream,							\
396	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
397} while (0)
398
399/* cmpl i32,dr32 */
400#define CMPid(i32, dr32) do {						\
401	if (dr32 == EAX){						\
402		emitm(&stream, 0x3d, 1);				\
403		emitm(&stream, i32, 4);					\
404	} else {							\
405		emitm(&stream, 0x81, 1);				\
406		emitm(&stream, (0x1f << 3) | (dr32 & 0x7), 1);		\
407		emitm(&stream, i32, 4);					\
408	}								\
409} while (0)
410
411/* jb off8 */
412#define JBb(off8) do {							\
413	emitm(&stream, 0x72, 1);					\
414	emitm(&stream, off8, 1);					\
415} while (0)
416
417/* jae off8 */
418#define JAEb(off8) do {							\
419	emitm(&stream, 0x73, 1);					\
420	emitm(&stream, off8, 1);					\
421} while (0)
422
423/* jne off8 */
424#define JNEb(off8) do {							\
425	emitm(&stream, 0x75, 1);					\
426	emitm(&stream, off8, 1);					\
427} while (0)
428
429/* ja off8 */
430#define JAb(off8) do {							\
431	emitm(&stream, 0x77, 1);					\
432	emitm(&stream, off8, 1);					\
433} while (0)
434
435/* jmp off32 */
436#define JMP(off32) do {							\
437	emitm(&stream, 0xe9, 1);					\
438	emitm(&stream, off32, 4);					\
439} while (0)
440
441/* xorl r32,r32 */
442#define ZEROrd(r32) do {						\
443	emitm(&stream, 0x31, 1);					\
444	emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1);	\
445} while (0)
446
447/*
448 * Conditional long jumps
449 */
450#define	JB	0x82
451#define	JAE	0x83
452#define	JE	0x84
453#define	JNE	0x85
454#define	JBE	0x86
455#define	JA	0x87
456
457#define	JCC(t, f) do {							\
458	if (ins->jt != 0 && ins->jf != 0) {				\
459		/* 5 is the size of the following jmp */		\
460		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
461		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
462		    stream.refs[stream.bpf_pc] + 5, 4);			\
463		JMP(stream.refs[stream.bpf_pc + ins->jf] -		\
464		    stream.refs[stream.bpf_pc]);			\
465	} else if (ins->jt != 0) {					\
466		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
467		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
468		    stream.refs[stream.bpf_pc], 4);			\
469	} else {							\
470		emitm(&stream, ((f) << 8) | 0x0f, 2);			\
471		emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] -	\
472		    stream.refs[stream.bpf_pc], 4);			\
473	}								\
474} while (0)
475
476#define	JUMP(off) do {							\
477	if ((off) != 0)							\
478		JMP(stream.refs[stream.bpf_pc + (off)] -		\
479		    stream.refs[stream.bpf_pc]);			\
480} while (0)
481
482#endif	/* _BPF_JIT_MACHDEP_H_ */
483