1/*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) 5 * Copyright (C) 2005-2016 Jung-uk Kim <jkim@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. Neither the name of the Politecnico di Torino nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 * 33 * $FreeBSD$ 34 */ 35 36#ifndef _BPF_JIT_MACHDEP_H_ 37#define _BPF_JIT_MACHDEP_H_ 38 39/* 40 * Registers 41 */ 42#define RAX 0 43#define RCX 1 44#define RDX 2 45#define RBX 3 46#define RSP 4 47#define RBP 5 48#define RSI 6 49#define RDI 7 50#define R8 0 51#define R9 1 52#define R10 2 53#define R11 3 54#define R12 4 55#define R13 5 56#define R14 6 57#define R15 7 58 59#define EAX 0 60#define ECX 1 61#define EDX 2 62#define EBX 3 63#define ESP 4 64#define EBP 5 65#define ESI 6 66#define EDI 7 67#define R8D 0 68#define R9D 1 69#define R10D 2 70#define R11D 3 71#define R12D 4 72#define R13D 5 73#define R14D 6 74#define R15D 7 75 76#define AX 0 77#define CX 1 78#define DX 2 79#define BX 3 80#define SP 4 81#define BP 5 82#define SI 6 83#define DI 7 84 85#define AL 0 86#define CL 1 87#define DL 2 88#define BL 3 89 90/* Optimization flags */ 91#define BPF_JIT_FRET 0x01 92#define BPF_JIT_FPKT 0x02 93#define BPF_JIT_FMEM 0x04 94#define BPF_JIT_FJMP 0x08 95#define BPF_JIT_FLEN 0x10 96 97#define BPF_JIT_FLAG_ALL \ 98 (BPF_JIT_FPKT | BPF_JIT_FMEM | BPF_JIT_FJMP | BPF_JIT_FLEN) 99 100/* A stream of native binary code */ 101typedef struct bpf_bin_stream { 102 /* Current native instruction pointer. */ 103 int cur_ip; 104 105 /* 106 * Current BPF instruction pointer, i.e. position in 107 * the BPF program reached by the jitter. 108 */ 109 int bpf_pc; 110 111 /* Instruction buffer, contains the generated native code. */ 112 char *ibuf; 113 114 /* Jumps reference table. */ 115 u_int *refs; 116} bpf_bin_stream; 117 118/* 119 * Prototype of the emit functions. 120 * 121 * Different emit functions are used to create the reference table and 122 * to generate the actual filtering code. This allows to have simpler 123 * instruction macros. 124 * The first parameter is the stream that will receive the data. 125 * The second one is a variable containing the data. 126 * The third one is the length, that can be 1, 2, or 4 since it is possible 127 * to emit a byte, a short, or a word at a time. 128 */ 129typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); 130 131/* 132 * Native instruction macros 133 */ 134 135/* movl i32,r32 */ 136#define MOVid(i32, r32) do { \ 137 emitm(&stream, (11 << 4) | (1 << 3) | (r32 & 0x7), 1); \ 138 emitm(&stream, i32, 4); \ 139} while (0) 140 141/* movq i64,r64 */ 142#define MOViq(i64, r64) do { \ 143 emitm(&stream, 0x48, 1); \ 144 emitm(&stream, (11 << 4) | (1 << 3) | (r64 & 0x7), 1); \ 145 emitm(&stream, i64, 4); \ 146 emitm(&stream, (i64 >> 32), 4); \ 147} while (0) 148 149/* movl sr32,dr32 */ 150#define MOVrd(sr32, dr32) do { \ 151 emitm(&stream, 0x89, 1); \ 152 emitm(&stream, \ 153 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 154} while (0) 155 156/* movl sr32,dr32 (dr32 = %r8-15d) */ 157#define MOVrd2(sr32, dr32) do { \ 158 emitm(&stream, 0x8941, 2); \ 159 emitm(&stream, \ 160 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 161} while (0) 162 163/* movl sr32,dr32 (sr32 = %r8-15d) */ 164#define MOVrd3(sr32, dr32) do { \ 165 emitm(&stream, 0x8944, 2); \ 166 emitm(&stream, \ 167 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 168} while (0) 169 170/* movq sr64,dr64 */ 171#define MOVrq(sr64, dr64) do { \ 172 emitm(&stream, 0x8948, 2); \ 173 emitm(&stream, \ 174 (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 175} while (0) 176 177/* movq sr64,dr64 (dr64 = %r8-15) */ 178#define MOVrq2(sr64, dr64) do { \ 179 emitm(&stream, 0x8949, 2); \ 180 emitm(&stream, \ 181 (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 182} while (0) 183 184/* movq sr64,dr64 (sr64 = %r8-15) */ 185#define MOVrq3(sr64, dr64) do { \ 186 emitm(&stream, 0x894c, 2); \ 187 emitm(&stream, \ 188 (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 189} while (0) 190 191/* movl (sr64,or64,1),dr32 */ 192#define MOVobd(sr64, or64, dr32) do { \ 193 emitm(&stream, 0x8b, 1); \ 194 emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1); \ 195 emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1); \ 196} while (0) 197 198/* movw (sr64,or64,1),dr16 */ 199#define MOVobw(sr64, or64, dr16) do { \ 200 emitm(&stream, 0x8b66, 2); \ 201 emitm(&stream, ((dr16 & 0x7) << 3) | 4, 1); \ 202 emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1); \ 203} while (0) 204 205/* movb (sr64,or64,1),dr8 */ 206#define MOVobb(sr64, or64, dr8) do { \ 207 emitm(&stream, 0x8a, 1); \ 208 emitm(&stream, ((dr8 & 0x7) << 3) | 4, 1); \ 209 emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1); \ 210} while (0) 211 212/* movl sr32,(dr64,or64,1) */ 213#define MOVomd(sr32, dr64, or64) do { \ 214 emitm(&stream, 0x89, 1); \ 215 emitm(&stream, ((sr32 & 0x7) << 3) | 4, 1); \ 216 emitm(&stream, ((or64 & 0x7) << 3) | (dr64 & 0x7), 1); \ 217} while (0) 218 219/* bswapl dr32 */ 220#define BSWAP(dr32) do { \ 221 emitm(&stream, 0xf, 1); \ 222 emitm(&stream, (0x19 << 3) | dr32, 1); \ 223} while (0) 224 225/* xchgb %al,%ah */ 226#define SWAP_AX() do { \ 227 emitm(&stream, 0xc486, 2); \ 228} while (0) 229 230/* pushq r64 */ 231#define PUSH(r64) do { \ 232 emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1); \ 233} while (0) 234 235/* leaveq */ 236#define LEAVE() do { \ 237 emitm(&stream, 0xc9, 1); \ 238} while (0) 239 240/* retq */ 241#define RET() do { \ 242 emitm(&stream, 0xc3, 1); \ 243} while (0) 244 245/* addl sr32,dr32 */ 246#define ADDrd(sr32, dr32) do { \ 247 emitm(&stream, 0x01, 1); \ 248 emitm(&stream, \ 249 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 250} while (0) 251 252/* addl i32,%eax */ 253#define ADD_EAXi(i32) do { \ 254 emitm(&stream, 0x05, 1); \ 255 emitm(&stream, i32, 4); \ 256} while (0) 257 258/* addl i8,r32 */ 259#define ADDib(i8, r32) do { \ 260 emitm(&stream, 0x83, 1); \ 261 emitm(&stream, (24 << 3) | r32, 1); \ 262 emitm(&stream, i8, 1); \ 263} while (0) 264 265/* subl sr32,dr32 */ 266#define SUBrd(sr32, dr32) do { \ 267 emitm(&stream, 0x29, 1); \ 268 emitm(&stream, \ 269 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 270} while (0) 271 272/* subl i32,%eax */ 273#define SUB_EAXi(i32) do { \ 274 emitm(&stream, 0x2d, 1); \ 275 emitm(&stream, i32, 4); \ 276} while (0) 277 278/* subq i8,r64 */ 279#define SUBib(i8, r64) do { \ 280 emitm(&stream, 0x8348, 2); \ 281 emitm(&stream, (29 << 3) | (r64 & 0x7), 1); \ 282 emitm(&stream, i8, 1); \ 283} while (0) 284 285/* mull r32 */ 286#define MULrd(r32) do { \ 287 emitm(&stream, 0xf7, 1); \ 288 emitm(&stream, (7 << 5) | (r32 & 0x7), 1); \ 289} while (0) 290 291/* divl r32 */ 292#define DIVrd(r32) do { \ 293 emitm(&stream, 0xf7, 1); \ 294 emitm(&stream, (15 << 4) | (r32 & 0x7), 1); \ 295} while (0) 296 297/* andb i8,r8 */ 298#define ANDib(i8, r8) do { \ 299 if (r8 == AL) { \ 300 emitm(&stream, 0x24, 1); \ 301 } else { \ 302 emitm(&stream, 0x80, 1); \ 303 emitm(&stream, (7 << 5) | r8, 1); \ 304 } \ 305 emitm(&stream, i8, 1); \ 306} while (0) 307 308/* andl i32,r32 */ 309#define ANDid(i32, r32) do { \ 310 if (r32 == EAX) { \ 311 emitm(&stream, 0x25, 1); \ 312 } else { \ 313 emitm(&stream, 0x81, 1); \ 314 emitm(&stream, (7 << 5) | r32, 1); \ 315 } \ 316 emitm(&stream, i32, 4); \ 317} while (0) 318 319/* andl sr32,dr32 */ 320#define ANDrd(sr32, dr32) do { \ 321 emitm(&stream, 0x21, 1); \ 322 emitm(&stream, \ 323 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 324} while (0) 325 326/* testl i32,r32 */ 327#define TESTid(i32, r32) do { \ 328 if (r32 == EAX) { \ 329 emitm(&stream, 0xa9, 1); \ 330 } else { \ 331 emitm(&stream, 0xf7, 1); \ 332 emitm(&stream, (3 << 6) | r32, 1); \ 333 } \ 334 emitm(&stream, i32, 4); \ 335} while (0) 336 337/* testl sr32,dr32 */ 338#define TESTrd(sr32, dr32) do { \ 339 emitm(&stream, 0x85, 1); \ 340 emitm(&stream, \ 341 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 342} while (0) 343 344/* orl sr32,dr32 */ 345#define ORrd(sr32, dr32) do { \ 346 emitm(&stream, 0x09, 1); \ 347 emitm(&stream, \ 348 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 349} while (0) 350 351/* orl i32,r32 */ 352#define ORid(i32, r32) do { \ 353 if (r32 == EAX) { \ 354 emitm(&stream, 0x0d, 1); \ 355 } else { \ 356 emitm(&stream, 0x81, 1); \ 357 emitm(&stream, (25 << 3) | r32, 1); \ 358 } \ 359 emitm(&stream, i32, 4); \ 360} while (0) 361 362/* xorl sr32,dr32 */ 363#define XORrd(sr32, dr32) do { \ 364 emitm(&stream, 0x31, 1); \ 365 emitm(&stream, \ 366 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 367} while (0) 368 369/* xorl i32,r32 */ 370#define XORid(i32, r32) do { \ 371 if (r32 == EAX) { \ 372 emitm(&stream, 0x35, 1); \ 373 } else { \ 374 emitm(&stream, 0x81, 1); \ 375 emitm(&stream, (25 << 3) | r32, 1); \ 376 } \ 377 emitm(&stream, i32, 4); \ 378} while (0) 379 380/* shll i8,r32 */ 381#define SHLib(i8, r32) do { \ 382 emitm(&stream, 0xc1, 1); \ 383 emitm(&stream, (7 << 5) | (r32 & 0x7), 1); \ 384 emitm(&stream, i8, 1); \ 385} while (0) 386 387/* shll %cl,dr32 */ 388#define SHL_CLrb(dr32) do { \ 389 emitm(&stream, 0xd3, 1); \ 390 emitm(&stream, (7 << 5) | (dr32 & 0x7), 1); \ 391} while (0) 392 393/* shrl i8,r32 */ 394#define SHRib(i8, r32) do { \ 395 emitm(&stream, 0xc1, 1); \ 396 emitm(&stream, (29 << 3) | (r32 & 0x7), 1); \ 397 emitm(&stream, i8, 1); \ 398} while (0) 399 400/* shrl %cl,dr32 */ 401#define SHR_CLrb(dr32) do { \ 402 emitm(&stream, 0xd3, 1); \ 403 emitm(&stream, (29 << 3) | (dr32 & 0x7), 1); \ 404} while (0) 405 406/* negl r32 */ 407#define NEGd(r32) do { \ 408 emitm(&stream, 0xf7, 1); \ 409 emitm(&stream, (27 << 3) | (r32 & 0x7), 1); \ 410} while (0) 411 412/* cmpl sr32,dr32 */ 413#define CMPrd(sr32, dr32) do { \ 414 emitm(&stream, 0x39, 1); \ 415 emitm(&stream, \ 416 (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1); \ 417} while (0) 418 419/* cmpl i32,dr32 */ 420#define CMPid(i32, dr32) do { \ 421 if (dr32 == EAX){ \ 422 emitm(&stream, 0x3d, 1); \ 423 emitm(&stream, i32, 4); \ 424 } else { \ 425 emitm(&stream, 0x81, 1); \ 426 emitm(&stream, (0x1f << 3) | (dr32 & 0x7), 1); \ 427 emitm(&stream, i32, 4); \ 428 } \ 429} while (0) 430 431/* jb off8 */ 432#define JBb(off8) do { \ 433 emitm(&stream, 0x72, 1); \ 434 emitm(&stream, off8, 1); \ 435} while (0) 436 437/* jae off8 */ 438#define JAEb(off8) do { \ 439 emitm(&stream, 0x73, 1); \ 440 emitm(&stream, off8, 1); \ 441} while (0) 442 443/* jne off8 */ 444#define JNEb(off8) do { \ 445 emitm(&stream, 0x75, 1); \ 446 emitm(&stream, off8, 1); \ 447} while (0) 448 449/* ja off8 */ 450#define JAb(off8) do { \ 451 emitm(&stream, 0x77, 1); \ 452 emitm(&stream, off8, 1); \ 453} while (0) 454 455/* jmp off32 */ 456#define JMP(off32) do { \ 457 emitm(&stream, 0xe9, 1); \ 458 emitm(&stream, off32, 4); \ 459} while (0) 460 461/* xorl r32,r32 */ 462#define ZEROrd(r32) do { \ 463 emitm(&stream, 0x31, 1); \ 464 emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1); \ 465} while (0) 466 467/* 468 * Conditional long jumps 469 */ 470#define JB 0x82 471#define JAE 0x83 472#define JE 0x84 473#define JNE 0x85 474#define JBE 0x86 475#define JA 0x87 476 477#define JCC(t, f) do { \ 478 if (ins->jt != 0 && ins->jf != 0) { \ 479 /* 5 is the size of the following jmp */ \ 480 emitm(&stream, ((t) << 8) | 0x0f, 2); \ 481 emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] - \ 482 stream.refs[stream.bpf_pc] + 5, 4); \ 483 JMP(stream.refs[stream.bpf_pc + ins->jf] - \ 484 stream.refs[stream.bpf_pc]); \ 485 } else if (ins->jt != 0) { \ 486 emitm(&stream, ((t) << 8) | 0x0f, 2); \ 487 emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] - \ 488 stream.refs[stream.bpf_pc], 4); \ 489 } else { \ 490 emitm(&stream, ((f) << 8) | 0x0f, 2); \ 491 emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] - \ 492 stream.refs[stream.bpf_pc], 4); \ 493 } \ 494} while (0) 495 496#define JUMP(off) do { \ 497 if ((off) != 0) \ 498 JMP(stream.refs[stream.bpf_pc + (off)] - \ 499 stream.refs[stream.bpf_pc]); \ 500} while (0) 501 502#endif /* _BPF_JIT_MACHDEP_H_ */ 503