bpf_jit_machdep.c revision 199603
1/*- 2 * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) 3 * Copyright (C) 2005-2009 Jung-uk Kim <jkim@FreeBSD.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the Politecnico di Torino nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: head/sys/amd64/amd64/bpf_jit_machdep.c 199603 2009-11-20 18:49:20Z jkim $"); 34 35#ifdef _KERNEL 36#include "opt_bpf.h" 37#include <sys/param.h> 38#include <sys/systm.h> 39#include <sys/kernel.h> 40#include <sys/socket.h> 41#include <sys/malloc.h> 42#include <net/if.h> 43#else 44#include <stdlib.h> 45#include <sys/mman.h> 46#include <sys/param.h> 47#endif 48 49#include <sys/types.h> 50 51#include <net/bpf.h> 52#include <net/bpf_jitter.h> 53 54#include <amd64/amd64/bpf_jit_machdep.h> 55 56bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); 57 58/* 59 * emit routine to update the jump table 60 */ 61static void 62emit_length(bpf_bin_stream *stream, __unused u_int value, u_int len) 63{ 64 65 (stream->refs)[stream->bpf_pc] += len; 66 stream->cur_ip += len; 67} 68 69/* 70 * emit routine to output the actual binary code 71 */ 72static void 73emit_code(bpf_bin_stream *stream, u_int value, u_int len) 74{ 75 76 switch (len) { 77 case 1: 78 stream->ibuf[stream->cur_ip] = (u_char)value; 79 stream->cur_ip++; 80 break; 81 82 case 2: 83 *((u_short *)(stream->ibuf + stream->cur_ip)) = (u_short)value; 84 stream->cur_ip += 2; 85 break; 86 87 case 4: 88 *((u_int *)(stream->ibuf + stream->cur_ip)) = value; 89 stream->cur_ip += 4; 90 break; 91 } 92 93 return; 94} 95 96/* 97 * Function that does the real stuff 98 */ 99bpf_filter_func 100bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) 101{ 102 bpf_bin_stream stream; 103 struct bpf_insn *ins; 104 u_int i, pass; 105 106 /* 107 * NOTE: do not modify the name of this variable, as it's used by 108 * the macros to emit code. 109 */ 110 emit_func emitm; 111 112 /* Allocate the reference table for the jumps */ 113#ifdef _KERNEL 114 stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT); 115#else 116 stream.refs = malloc((nins + 1) * sizeof(u_int)); 117#endif 118 if (stream.refs == NULL) 119 return (NULL); 120 121 /* Reset the reference table */ 122 for (i = 0; i < nins + 1; i++) 123 stream.refs[i] = 0; 124 125 stream.cur_ip = 0; 126 stream.bpf_pc = 0; 127 128 /* 129 * the first pass will emit the lengths of the instructions 130 * to create the reference table 131 */ 132 emitm = emit_length; 133 134 pass = 0; 135 for (;;) { 136 ins = prog; 137 138 /* create the procedure header */ 139 PUSH(RBP); 140 MOVrq(RSP, RBP); 141 SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP); 142 MOVrq2(RDI, R8); 143 MOVrd2(ESI, R9D); 144 MOVrd(EDX, EDI); 145 146 for (i = 0; i < nins; i++) { 147 stream.bpf_pc++; 148 149 switch (ins->code) { 150 default: 151#ifdef _KERNEL 152 return (NULL); 153#else 154 abort(); 155#endif 156 157 case BPF_RET|BPF_K: 158 MOVid(ins->k, EAX); 159 LEAVE_RET(); 160 break; 161 162 case BPF_RET|BPF_A: 163 LEAVE_RET(); 164 break; 165 166 case BPF_LD|BPF_W|BPF_ABS: 167 MOVid(ins->k, ESI); 168 CMPrd(EDI, ESI); 169 JAb(12); 170 MOVrd(EDI, ECX); 171 SUBrd(ESI, ECX); 172 CMPid(sizeof(int32_t), ECX); 173 JAEb(4); 174 ZEROrd(EAX); 175 LEAVE_RET(); 176 MOVrq3(R8, RCX); 177 MOVobd(RCX, RSI, EAX); 178 BSWAP(EAX); 179 break; 180 181 case BPF_LD|BPF_H|BPF_ABS: 182 ZEROrd(EAX); 183 MOVid(ins->k, ESI); 184 CMPrd(EDI, ESI); 185 JAb(12); 186 MOVrd(EDI, ECX); 187 SUBrd(ESI, ECX); 188 CMPid(sizeof(int16_t), ECX); 189 JAEb(2); 190 LEAVE_RET(); 191 MOVrq3(R8, RCX); 192 MOVobw(RCX, RSI, AX); 193 SWAP_AX(); 194 break; 195 196 case BPF_LD|BPF_B|BPF_ABS: 197 ZEROrd(EAX); 198 MOVid(ins->k, ESI); 199 CMPrd(EDI, ESI); 200 JBb(2); 201 LEAVE_RET(); 202 MOVrq3(R8, RCX); 203 MOVobb(RCX, RSI, AL); 204 break; 205 206 case BPF_LD|BPF_W|BPF_LEN: 207 MOVrd3(R9D, EAX); 208 break; 209 210 case BPF_LDX|BPF_W|BPF_LEN: 211 MOVrd3(R9D, EDX); 212 break; 213 214 case BPF_LD|BPF_W|BPF_IND: 215 CMPrd(EDI, EDX); 216 JAb(27); 217 MOVid(ins->k, ESI); 218 MOVrd(EDI, ECX); 219 SUBrd(EDX, ECX); 220 CMPrd(ESI, ECX); 221 JBb(14); 222 ADDrd(EDX, ESI); 223 MOVrd(EDI, ECX); 224 SUBrd(ESI, ECX); 225 CMPid(sizeof(int32_t), ECX); 226 JAEb(4); 227 ZEROrd(EAX); 228 LEAVE_RET(); 229 MOVrq3(R8, RCX); 230 MOVobd(RCX, RSI, EAX); 231 BSWAP(EAX); 232 break; 233 234 case BPF_LD|BPF_H|BPF_IND: 235 ZEROrd(EAX); 236 CMPrd(EDI, EDX); 237 JAb(27); 238 MOVid(ins->k, ESI); 239 MOVrd(EDI, ECX); 240 SUBrd(EDX, ECX); 241 CMPrd(ESI, ECX); 242 JBb(14); 243 ADDrd(EDX, ESI); 244 MOVrd(EDI, ECX); 245 SUBrd(ESI, ECX); 246 CMPid(sizeof(int16_t), ECX); 247 JAEb(2); 248 LEAVE_RET(); 249 MOVrq3(R8, RCX); 250 MOVobw(RCX, RSI, AX); 251 SWAP_AX(); 252 break; 253 254 case BPF_LD|BPF_B|BPF_IND: 255 ZEROrd(EAX); 256 CMPrd(EDI, EDX); 257 JAEb(13); 258 MOVid(ins->k, ESI); 259 MOVrd(EDI, ECX); 260 SUBrd(EDX, ECX); 261 CMPrd(ESI, ECX); 262 JAb(2); 263 LEAVE_RET(); 264 MOVrq3(R8, RCX); 265 ADDrd(EDX, ESI); 266 MOVobb(RCX, RSI, AL); 267 break; 268 269 case BPF_LDX|BPF_MSH|BPF_B: 270 MOVid(ins->k, ESI); 271 CMPrd(EDI, ESI); 272 JBb(4); 273 ZEROrd(EAX); 274 LEAVE_RET(); 275 ZEROrd(EDX); 276 MOVrq3(R8, RCX); 277 MOVobb(RCX, RSI, DL); 278 ANDib(0x0f, DL); 279 SHLib(2, EDX); 280 break; 281 282 case BPF_LD|BPF_IMM: 283 MOVid(ins->k, EAX); 284 break; 285 286 case BPF_LDX|BPF_IMM: 287 MOVid(ins->k, EDX); 288 break; 289 290 case BPF_LD|BPF_MEM: 291 MOVid(ins->k * sizeof(uint32_t), ESI); 292 MOVobd(RSP, RSI, EAX); 293 break; 294 295 case BPF_LDX|BPF_MEM: 296 MOVid(ins->k * sizeof(uint32_t), ESI); 297 MOVobd(RSP, RSI, EDX); 298 break; 299 300 case BPF_ST: 301 /* 302 * XXX this command and the following could 303 * be optimized if the previous instruction 304 * was already of this type 305 */ 306 MOVid(ins->k * sizeof(uint32_t), ESI); 307 MOVomd(EAX, RSP, RSI); 308 break; 309 310 case BPF_STX: 311 MOVid(ins->k * sizeof(uint32_t), ESI); 312 MOVomd(EDX, RSP, RSI); 313 break; 314 315 case BPF_JMP|BPF_JA: 316 JMP(stream.refs[stream.bpf_pc + ins->k] - 317 stream.refs[stream.bpf_pc]); 318 break; 319 320 case BPF_JMP|BPF_JGT|BPF_K: 321 if (ins->jt == 0 && ins->jf == 0) 322 break; 323 CMPid(ins->k, EAX); 324 JCC(JA, JBE); 325 break; 326 327 case BPF_JMP|BPF_JGE|BPF_K: 328 if (ins->jt == 0 && ins->jf == 0) 329 break; 330 CMPid(ins->k, EAX); 331 JCC(JAE, JB); 332 break; 333 334 case BPF_JMP|BPF_JEQ|BPF_K: 335 if (ins->jt == 0 && ins->jf == 0) 336 break; 337 CMPid(ins->k, EAX); 338 JCC(JE, JNE); 339 break; 340 341 case BPF_JMP|BPF_JSET|BPF_K: 342 if (ins->jt == 0 && ins->jf == 0) 343 break; 344 TESTid(ins->k, EAX); 345 JCC(JNE, JE); 346 break; 347 348 case BPF_JMP|BPF_JGT|BPF_X: 349 if (ins->jt == 0 && ins->jf == 0) 350 break; 351 CMPrd(EDX, EAX); 352 JCC(JA, JBE); 353 break; 354 355 case BPF_JMP|BPF_JGE|BPF_X: 356 if (ins->jt == 0 && ins->jf == 0) 357 break; 358 CMPrd(EDX, EAX); 359 JCC(JAE, JB); 360 break; 361 362 case BPF_JMP|BPF_JEQ|BPF_X: 363 if (ins->jt == 0 && ins->jf == 0) 364 break; 365 CMPrd(EDX, EAX); 366 JCC(JE, JNE); 367 break; 368 369 case BPF_JMP|BPF_JSET|BPF_X: 370 if (ins->jt == 0 && ins->jf == 0) 371 break; 372 TESTrd(EDX, EAX); 373 JCC(JNE, JE); 374 break; 375 376 case BPF_ALU|BPF_ADD|BPF_X: 377 ADDrd(EDX, EAX); 378 break; 379 380 case BPF_ALU|BPF_SUB|BPF_X: 381 SUBrd(EDX, EAX); 382 break; 383 384 case BPF_ALU|BPF_MUL|BPF_X: 385 MOVrd(EDX, ECX); 386 MULrd(EDX); 387 MOVrd(ECX, EDX); 388 break; 389 390 case BPF_ALU|BPF_DIV|BPF_X: 391 TESTrd(EDX, EDX); 392 JNEb(4); 393 ZEROrd(EAX); 394 LEAVE_RET(); 395 MOVrd(EDX, ECX); 396 ZEROrd(EDX); 397 DIVrd(ECX); 398 MOVrd(ECX, EDX); 399 break; 400 401 case BPF_ALU|BPF_AND|BPF_X: 402 ANDrd(EDX, EAX); 403 break; 404 405 case BPF_ALU|BPF_OR|BPF_X: 406 ORrd(EDX, EAX); 407 break; 408 409 case BPF_ALU|BPF_LSH|BPF_X: 410 MOVrd(EDX, ECX); 411 SHL_CLrb(EAX); 412 break; 413 414 case BPF_ALU|BPF_RSH|BPF_X: 415 MOVrd(EDX, ECX); 416 SHR_CLrb(EAX); 417 break; 418 419 case BPF_ALU|BPF_ADD|BPF_K: 420 ADD_EAXi(ins->k); 421 break; 422 423 case BPF_ALU|BPF_SUB|BPF_K: 424 SUB_EAXi(ins->k); 425 break; 426 427 case BPF_ALU|BPF_MUL|BPF_K: 428 MOVrd(EDX, ECX); 429 MOVid(ins->k, EDX); 430 MULrd(EDX); 431 MOVrd(ECX, EDX); 432 break; 433 434 case BPF_ALU|BPF_DIV|BPF_K: 435 MOVrd(EDX, ECX); 436 ZEROrd(EDX); 437 MOVid(ins->k, ESI); 438 DIVrd(ESI); 439 MOVrd(ECX, EDX); 440 break; 441 442 case BPF_ALU|BPF_AND|BPF_K: 443 ANDid(ins->k, EAX); 444 break; 445 446 case BPF_ALU|BPF_OR|BPF_K: 447 ORid(ins->k, EAX); 448 break; 449 450 case BPF_ALU|BPF_LSH|BPF_K: 451 SHLib((ins->k) & 0xff, EAX); 452 break; 453 454 case BPF_ALU|BPF_RSH|BPF_K: 455 SHRib((ins->k) & 0xff, EAX); 456 break; 457 458 case BPF_ALU|BPF_NEG: 459 NEGd(EAX); 460 break; 461 462 case BPF_MISC|BPF_TAX: 463 MOVrd(EAX, EDX); 464 break; 465 466 case BPF_MISC|BPF_TXA: 467 MOVrd(EDX, EAX); 468 break; 469 } 470 ins++; 471 } 472 473 pass++; 474 if (pass >= 2) { 475#ifndef _KERNEL 476 if (mprotect(stream.ibuf, stream.cur_ip, 477 PROT_READ | PROT_EXEC) != 0) { 478 munmap(stream.ibuf, stream.cur_ip); 479 stream.ibuf = NULL; 480 } 481#endif 482 *size = stream.cur_ip; 483 break; 484 } 485 486#ifdef _KERNEL 487 stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT); 488 if (stream.ibuf == NULL) 489 break; 490#else 491 stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE, 492 MAP_ANON, -1, 0); 493 if (stream.ibuf == MAP_FAILED) { 494 stream.ibuf = NULL; 495 break; 496 } 497#endif 498 499 /* 500 * modify the reference table to contain the offsets and 501 * not the lengths of the instructions 502 */ 503 for (i = 1; i < nins + 1; i++) 504 stream.refs[i] += stream.refs[i - 1]; 505 506 /* Reset the counters */ 507 stream.cur_ip = 0; 508 stream.bpf_pc = 0; 509 510 /* the second pass creates the actual code */ 511 emitm = emit_code; 512 } 513 514 /* 515 * the reference table is needed only during compilation, 516 * now we can free it 517 */ 518#ifdef _KERNEL 519 free(stream.refs, M_BPFJIT); 520#else 521 free(stream.refs); 522#endif 523 524 return ((bpf_filter_func)stream.ibuf); 525} 526