bpf_jit_machdep.c revision 199615
1/*- 2 * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) 3 * Copyright (C) 2005-2009 Jung-uk Kim <jkim@FreeBSD.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the Politecnico di Torino nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: head/sys/amd64/amd64/bpf_jit_machdep.c 199615 2009-11-20 21:12:40Z jkim $"); 34 35#ifdef _KERNEL 36#include "opt_bpf.h" 37#include <sys/param.h> 38#include <sys/systm.h> 39#include <sys/kernel.h> 40#include <sys/socket.h> 41#include <sys/malloc.h> 42#include <net/if.h> 43#else 44#include <stdlib.h> 45#include <string.h> 46#include <sys/mman.h> 47#include <sys/param.h> 48#endif 49 50#include <sys/types.h> 51 52#include <net/bpf.h> 53#include <net/bpf_jitter.h> 54 55#include <amd64/amd64/bpf_jit_machdep.h> 56 57bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); 58 59/* 60 * emit routine to update the jump table 61 */ 62static void 63emit_length(bpf_bin_stream *stream, __unused u_int value, u_int len) 64{ 65 66 (stream->refs)[stream->bpf_pc] += len; 67 stream->cur_ip += len; 68} 69 70/* 71 * emit routine to output the actual binary code 72 */ 73static void 74emit_code(bpf_bin_stream *stream, u_int value, u_int len) 75{ 76 77 switch (len) { 78 case 1: 79 stream->ibuf[stream->cur_ip] = (u_char)value; 80 stream->cur_ip++; 81 break; 82 83 case 2: 84 *((u_short *)(stream->ibuf + stream->cur_ip)) = (u_short)value; 85 stream->cur_ip += 2; 86 break; 87 88 case 4: 89 *((u_int *)(stream->ibuf + stream->cur_ip)) = value; 90 stream->cur_ip += 4; 91 break; 92 } 93 94 return; 95} 96 97/* 98 * Function that does the real stuff 99 */ 100bpf_filter_func 101bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) 102{ 103 bpf_bin_stream stream; 104 struct bpf_insn *ins; 105 u_int i, pass; 106 107 /* 108 * NOTE: Do not modify the name of this variable, as it's used by 109 * the macros to emit code. 110 */ 111 emit_func emitm; 112 113 /* Allocate the reference table for the jumps. */ 114#ifdef _KERNEL 115 stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, 116 M_NOWAIT | M_ZERO); 117#else 118 stream.refs = malloc((nins + 1) * sizeof(u_int)); 119#endif 120 if (stream.refs == NULL) 121 return (NULL); 122#ifndef _KERNEL 123 memset(stream.refs, 0, (nins + 1) * sizeof(u_int)); 124#endif 125 126 stream.cur_ip = 0; 127 stream.bpf_pc = 0; 128 stream.ibuf = NULL; 129 130 /* 131 * The first pass will emit the lengths of the instructions 132 * to create the reference table. 133 */ 134 emitm = emit_length; 135 136 for (pass = 0; pass < 2; pass++) { 137 ins = prog; 138 139 /* Create the procedure header. */ 140 PUSH(RBP); 141 MOVrq(RSP, RBP); 142 SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP); 143 MOVrq2(RDI, R8); 144 MOVrd2(ESI, R9D); 145 MOVrd(EDX, EDI); 146 147 for (i = 0; i < nins; i++) { 148 stream.bpf_pc++; 149 150 switch (ins->code) { 151 default: 152#ifdef _KERNEL 153 return (NULL); 154#else 155 abort(); 156#endif 157 158 case BPF_RET|BPF_K: 159 MOVid(ins->k, EAX); 160 LEAVE_RET(); 161 break; 162 163 case BPF_RET|BPF_A: 164 LEAVE_RET(); 165 break; 166 167 case BPF_LD|BPF_W|BPF_ABS: 168 MOVid(ins->k, ESI); 169 CMPrd(EDI, ESI); 170 JAb(12); 171 MOVrd(EDI, ECX); 172 SUBrd(ESI, ECX); 173 CMPid(sizeof(int32_t), ECX); 174 JAEb(4); 175 ZEROrd(EAX); 176 LEAVE_RET(); 177 MOVrq3(R8, RCX); 178 MOVobd(RCX, RSI, EAX); 179 BSWAP(EAX); 180 break; 181 182 case BPF_LD|BPF_H|BPF_ABS: 183 ZEROrd(EAX); 184 MOVid(ins->k, ESI); 185 CMPrd(EDI, ESI); 186 JAb(12); 187 MOVrd(EDI, ECX); 188 SUBrd(ESI, ECX); 189 CMPid(sizeof(int16_t), ECX); 190 JAEb(2); 191 LEAVE_RET(); 192 MOVrq3(R8, RCX); 193 MOVobw(RCX, RSI, AX); 194 SWAP_AX(); 195 break; 196 197 case BPF_LD|BPF_B|BPF_ABS: 198 ZEROrd(EAX); 199 MOVid(ins->k, ESI); 200 CMPrd(EDI, ESI); 201 JBb(2); 202 LEAVE_RET(); 203 MOVrq3(R8, RCX); 204 MOVobb(RCX, RSI, AL); 205 break; 206 207 case BPF_LD|BPF_W|BPF_LEN: 208 MOVrd3(R9D, EAX); 209 break; 210 211 case BPF_LDX|BPF_W|BPF_LEN: 212 MOVrd3(R9D, EDX); 213 break; 214 215 case BPF_LD|BPF_W|BPF_IND: 216 CMPrd(EDI, EDX); 217 JAb(27); 218 MOVid(ins->k, ESI); 219 MOVrd(EDI, ECX); 220 SUBrd(EDX, ECX); 221 CMPrd(ESI, ECX); 222 JBb(14); 223 ADDrd(EDX, ESI); 224 MOVrd(EDI, ECX); 225 SUBrd(ESI, ECX); 226 CMPid(sizeof(int32_t), ECX); 227 JAEb(4); 228 ZEROrd(EAX); 229 LEAVE_RET(); 230 MOVrq3(R8, RCX); 231 MOVobd(RCX, RSI, EAX); 232 BSWAP(EAX); 233 break; 234 235 case BPF_LD|BPF_H|BPF_IND: 236 ZEROrd(EAX); 237 CMPrd(EDI, EDX); 238 JAb(27); 239 MOVid(ins->k, ESI); 240 MOVrd(EDI, ECX); 241 SUBrd(EDX, ECX); 242 CMPrd(ESI, ECX); 243 JBb(14); 244 ADDrd(EDX, ESI); 245 MOVrd(EDI, ECX); 246 SUBrd(ESI, ECX); 247 CMPid(sizeof(int16_t), ECX); 248 JAEb(2); 249 LEAVE_RET(); 250 MOVrq3(R8, RCX); 251 MOVobw(RCX, RSI, AX); 252 SWAP_AX(); 253 break; 254 255 case BPF_LD|BPF_B|BPF_IND: 256 ZEROrd(EAX); 257 CMPrd(EDI, EDX); 258 JAEb(13); 259 MOVid(ins->k, ESI); 260 MOVrd(EDI, ECX); 261 SUBrd(EDX, ECX); 262 CMPrd(ESI, ECX); 263 JAb(2); 264 LEAVE_RET(); 265 MOVrq3(R8, RCX); 266 ADDrd(EDX, ESI); 267 MOVobb(RCX, RSI, AL); 268 break; 269 270 case BPF_LDX|BPF_MSH|BPF_B: 271 MOVid(ins->k, ESI); 272 CMPrd(EDI, ESI); 273 JBb(4); 274 ZEROrd(EAX); 275 LEAVE_RET(); 276 ZEROrd(EDX); 277 MOVrq3(R8, RCX); 278 MOVobb(RCX, RSI, DL); 279 ANDib(0x0f, DL); 280 SHLib(2, EDX); 281 break; 282 283 case BPF_LD|BPF_IMM: 284 MOVid(ins->k, EAX); 285 break; 286 287 case BPF_LDX|BPF_IMM: 288 MOVid(ins->k, EDX); 289 break; 290 291 case BPF_LD|BPF_MEM: 292 MOVid(ins->k * sizeof(uint32_t), ESI); 293 MOVobd(RSP, RSI, EAX); 294 break; 295 296 case BPF_LDX|BPF_MEM: 297 MOVid(ins->k * sizeof(uint32_t), ESI); 298 MOVobd(RSP, RSI, EDX); 299 break; 300 301 case BPF_ST: 302 /* 303 * XXX this command and the following could 304 * be optimized if the previous instruction 305 * was already of this type 306 */ 307 MOVid(ins->k * sizeof(uint32_t), ESI); 308 MOVomd(EAX, RSP, RSI); 309 break; 310 311 case BPF_STX: 312 MOVid(ins->k * sizeof(uint32_t), ESI); 313 MOVomd(EDX, RSP, RSI); 314 break; 315 316 case BPF_JMP|BPF_JA: 317 JMP(stream.refs[stream.bpf_pc + ins->k] - 318 stream.refs[stream.bpf_pc]); 319 break; 320 321 case BPF_JMP|BPF_JGT|BPF_K: 322 if (ins->jt == 0 && ins->jf == 0) 323 break; 324 CMPid(ins->k, EAX); 325 JCC(JA, JBE); 326 break; 327 328 case BPF_JMP|BPF_JGE|BPF_K: 329 if (ins->jt == 0 && ins->jf == 0) 330 break; 331 CMPid(ins->k, EAX); 332 JCC(JAE, JB); 333 break; 334 335 case BPF_JMP|BPF_JEQ|BPF_K: 336 if (ins->jt == 0 && ins->jf == 0) 337 break; 338 CMPid(ins->k, EAX); 339 JCC(JE, JNE); 340 break; 341 342 case BPF_JMP|BPF_JSET|BPF_K: 343 if (ins->jt == 0 && ins->jf == 0) 344 break; 345 TESTid(ins->k, EAX); 346 JCC(JNE, JE); 347 break; 348 349 case BPF_JMP|BPF_JGT|BPF_X: 350 if (ins->jt == 0 && ins->jf == 0) 351 break; 352 CMPrd(EDX, EAX); 353 JCC(JA, JBE); 354 break; 355 356 case BPF_JMP|BPF_JGE|BPF_X: 357 if (ins->jt == 0 && ins->jf == 0) 358 break; 359 CMPrd(EDX, EAX); 360 JCC(JAE, JB); 361 break; 362 363 case BPF_JMP|BPF_JEQ|BPF_X: 364 if (ins->jt == 0 && ins->jf == 0) 365 break; 366 CMPrd(EDX, EAX); 367 JCC(JE, JNE); 368 break; 369 370 case BPF_JMP|BPF_JSET|BPF_X: 371 if (ins->jt == 0 && ins->jf == 0) 372 break; 373 TESTrd(EDX, EAX); 374 JCC(JNE, JE); 375 break; 376 377 case BPF_ALU|BPF_ADD|BPF_X: 378 ADDrd(EDX, EAX); 379 break; 380 381 case BPF_ALU|BPF_SUB|BPF_X: 382 SUBrd(EDX, EAX); 383 break; 384 385 case BPF_ALU|BPF_MUL|BPF_X: 386 MOVrd(EDX, ECX); 387 MULrd(EDX); 388 MOVrd(ECX, EDX); 389 break; 390 391 case BPF_ALU|BPF_DIV|BPF_X: 392 TESTrd(EDX, EDX); 393 JNEb(4); 394 ZEROrd(EAX); 395 LEAVE_RET(); 396 MOVrd(EDX, ECX); 397 ZEROrd(EDX); 398 DIVrd(ECX); 399 MOVrd(ECX, EDX); 400 break; 401 402 case BPF_ALU|BPF_AND|BPF_X: 403 ANDrd(EDX, EAX); 404 break; 405 406 case BPF_ALU|BPF_OR|BPF_X: 407 ORrd(EDX, EAX); 408 break; 409 410 case BPF_ALU|BPF_LSH|BPF_X: 411 MOVrd(EDX, ECX); 412 SHL_CLrb(EAX); 413 break; 414 415 case BPF_ALU|BPF_RSH|BPF_X: 416 MOVrd(EDX, ECX); 417 SHR_CLrb(EAX); 418 break; 419 420 case BPF_ALU|BPF_ADD|BPF_K: 421 ADD_EAXi(ins->k); 422 break; 423 424 case BPF_ALU|BPF_SUB|BPF_K: 425 SUB_EAXi(ins->k); 426 break; 427 428 case BPF_ALU|BPF_MUL|BPF_K: 429 MOVrd(EDX, ECX); 430 MOVid(ins->k, EDX); 431 MULrd(EDX); 432 MOVrd(ECX, EDX); 433 break; 434 435 case BPF_ALU|BPF_DIV|BPF_K: 436 MOVrd(EDX, ECX); 437 ZEROrd(EDX); 438 MOVid(ins->k, ESI); 439 DIVrd(ESI); 440 MOVrd(ECX, EDX); 441 break; 442 443 case BPF_ALU|BPF_AND|BPF_K: 444 ANDid(ins->k, EAX); 445 break; 446 447 case BPF_ALU|BPF_OR|BPF_K: 448 ORid(ins->k, EAX); 449 break; 450 451 case BPF_ALU|BPF_LSH|BPF_K: 452 SHLib((ins->k) & 0xff, EAX); 453 break; 454 455 case BPF_ALU|BPF_RSH|BPF_K: 456 SHRib((ins->k) & 0xff, EAX); 457 break; 458 459 case BPF_ALU|BPF_NEG: 460 NEGd(EAX); 461 break; 462 463 case BPF_MISC|BPF_TAX: 464 MOVrd(EAX, EDX); 465 break; 466 467 case BPF_MISC|BPF_TXA: 468 MOVrd(EDX, EAX); 469 break; 470 } 471 ins++; 472 } 473 474 if (pass > 0) 475 continue; 476 477 *size = stream.cur_ip; 478#ifdef _KERNEL 479 stream.ibuf = malloc(*size, M_BPFJIT, M_NOWAIT); 480 if (stream.ibuf == NULL) 481 break; 482#else 483 stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE, 484 MAP_ANON, -1, 0); 485 if (stream.ibuf == MAP_FAILED) { 486 stream.ibuf = NULL; 487 break; 488 } 489#endif 490 491 /* 492 * Modify the reference table to contain the offsets and 493 * not the lengths of the instructions. 494 */ 495 for (i = 1; i < nins + 1; i++) 496 stream.refs[i] += stream.refs[i - 1]; 497 498 /* Reset the counters. */ 499 stream.cur_ip = 0; 500 stream.bpf_pc = 0; 501 502 /* The second pass creates the actual code. */ 503 emitm = emit_code; 504 } 505 506 /* 507 * The reference table is needed only during compilation, 508 * now we can free it. 509 */ 510#ifdef _KERNEL 511 free(stream.refs, M_BPFJIT); 512#else 513 free(stream.refs); 514 if (stream.ibuf != NULL && 515 mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) { 516 munmap(stream.ibuf, *size); 517 stream.ibuf = NULL; 518 } 519#endif 520 521 return ((bpf_filter_func)stream.ibuf); 522} 523