1153151Sjkim/*- 2181648Sjkim * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) 3199492Sjkim * Copyright (C) 2005-2009 Jung-uk Kim <jkim@FreeBSD.org> 4153151Sjkim * All rights reserved. 5153151Sjkim * 6153151Sjkim * Redistribution and use in source and binary forms, with or without 7153151Sjkim * modification, are permitted provided that the following conditions 8153151Sjkim * are met: 9153151Sjkim * 10153151Sjkim * 1. Redistributions of source code must retain the above copyright 11153151Sjkim * notice, this list of conditions and the following disclaimer. 12153151Sjkim * 2. Redistributions in binary form must reproduce the above copyright 13153151Sjkim * notice, this list of conditions and the following disclaimer in the 14153151Sjkim * documentation and/or other materials provided with the distribution. 15153151Sjkim * 3. Neither the name of the Politecnico di Torino nor the names of its 16153151Sjkim * contributors may be used to endorse or promote products derived from 17153151Sjkim * this software without specific prior written permission. 18153151Sjkim * 19153151Sjkim * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20153151Sjkim * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21153151Sjkim * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22153151Sjkim * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23153151Sjkim * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24153151Sjkim * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25153151Sjkim * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26182173Sjkim * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27153151Sjkim * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28153151Sjkim * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29153151Sjkim * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30153151Sjkim */ 31153151Sjkim 32153151Sjkim#include <sys/cdefs.h> 33153151Sjkim__FBSDID("$FreeBSD$"); 34153151Sjkim 35181846Sjkim#ifdef _KERNEL 36153151Sjkim#include "opt_bpf.h" 37153151Sjkim#include <sys/param.h> 38153151Sjkim#include <sys/systm.h> 39153151Sjkim#include <sys/kernel.h> 40153151Sjkim#include <sys/socket.h> 41153151Sjkim#include <sys/malloc.h> 42181846Sjkim#include <net/if.h> 43181846Sjkim#else 44181846Sjkim#include <stdlib.h> 45199615Sjkim#include <string.h> 46199492Sjkim#include <sys/mman.h> 47199492Sjkim#include <sys/param.h> 48181846Sjkim#endif 49153151Sjkim 50181846Sjkim#include <sys/types.h> 51181846Sjkim 52153151Sjkim#include <net/bpf.h> 53153151Sjkim#include <net/bpf_jitter.h> 54153151Sjkim 55153151Sjkim#include <i386/i386/bpf_jit_machdep.h> 56153151Sjkim 57199603Sjkimbpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); 58153151Sjkim 59153151Sjkim/* 60199619Sjkim * Emit routine to update the jump table. 61153151Sjkim */ 62153151Sjkimstatic void 63181846Sjkimemit_length(bpf_bin_stream *stream, __unused u_int value, u_int len) 64153151Sjkim{ 65153151Sjkim 66199619Sjkim if (stream->refs != NULL) 67199619Sjkim (stream->refs)[stream->bpf_pc] += len; 68153151Sjkim stream->cur_ip += len; 69153151Sjkim} 70153151Sjkim 71153151Sjkim/* 72199619Sjkim * Emit routine to output the actual binary code. 73153151Sjkim */ 74153151Sjkimstatic void 75153151Sjkimemit_code(bpf_bin_stream *stream, u_int value, u_int len) 76153151Sjkim{ 77153151Sjkim 78153151Sjkim switch (len) { 79153151Sjkim case 1: 80153151Sjkim stream->ibuf[stream->cur_ip] = (u_char)value; 81153151Sjkim stream->cur_ip++; 82153151Sjkim break; 83153151Sjkim 84153151Sjkim case 2: 85153151Sjkim *((u_short *)(stream->ibuf + stream->cur_ip)) = (u_short)value; 86153151Sjkim stream->cur_ip += 2; 87153151Sjkim break; 88153151Sjkim 89153151Sjkim case 4: 90153151Sjkim *((u_int *)(stream->ibuf + stream->cur_ip)) = value; 91153151Sjkim stream->cur_ip += 4; 92153151Sjkim break; 93153151Sjkim } 94153151Sjkim 95153151Sjkim return; 96153151Sjkim} 97153151Sjkim 98153151Sjkim/* 99199619Sjkim * Scan the filter program and find possible optimization. 100153151Sjkim */ 101199619Sjkimstatic int 102199619Sjkimbpf_jit_optimize(struct bpf_insn *prog, u_int nins) 103199619Sjkim{ 104199619Sjkim int flags; 105199619Sjkim u_int i; 106199619Sjkim 107199619Sjkim /* Do we return immediately? */ 108199619Sjkim if (BPF_CLASS(prog[0].code) == BPF_RET) 109199721Sjkim return (BPF_JIT_FRET); 110199619Sjkim 111199619Sjkim for (flags = 0, i = 0; i < nins; i++) { 112199721Sjkim switch (prog[i].code) { 113199721Sjkim case BPF_LD|BPF_W|BPF_ABS: 114199721Sjkim case BPF_LD|BPF_H|BPF_ABS: 115199721Sjkim case BPF_LD|BPF_B|BPF_ABS: 116199721Sjkim case BPF_LD|BPF_W|BPF_IND: 117199721Sjkim case BPF_LD|BPF_H|BPF_IND: 118199721Sjkim case BPF_LD|BPF_B|BPF_IND: 119199721Sjkim case BPF_LDX|BPF_MSH|BPF_B: 120199721Sjkim flags |= BPF_JIT_FPKT; 121199721Sjkim break; 122199721Sjkim case BPF_LD|BPF_MEM: 123199721Sjkim case BPF_LDX|BPF_MEM: 124199721Sjkim case BPF_ST: 125199721Sjkim case BPF_STX: 126199721Sjkim flags |= BPF_JIT_FMEM; 127199721Sjkim break; 128199721Sjkim case BPF_JMP|BPF_JA: 129199721Sjkim case BPF_JMP|BPF_JGT|BPF_K: 130199721Sjkim case BPF_JMP|BPF_JGE|BPF_K: 131199721Sjkim case BPF_JMP|BPF_JEQ|BPF_K: 132199721Sjkim case BPF_JMP|BPF_JSET|BPF_K: 133199721Sjkim case BPF_JMP|BPF_JGT|BPF_X: 134199721Sjkim case BPF_JMP|BPF_JGE|BPF_X: 135199721Sjkim case BPF_JMP|BPF_JEQ|BPF_X: 136199721Sjkim case BPF_JMP|BPF_JSET|BPF_X: 137199721Sjkim flags |= BPF_JIT_FJMP; 138199721Sjkim break; 139199721Sjkim case BPF_ALU|BPF_DIV|BPF_K: 140199721Sjkim flags |= BPF_JIT_FADK; 141199721Sjkim break; 142199721Sjkim } 143199619Sjkim if (flags == BPF_JIT_FLAG_ALL) 144199619Sjkim break; 145199619Sjkim } 146199619Sjkim 147199619Sjkim return (flags); 148199619Sjkim} 149199619Sjkim 150199619Sjkim/* 151199619Sjkim * Function that does the real stuff. 152199619Sjkim */ 153153151Sjkimbpf_filter_func 154199603Sjkimbpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) 155153151Sjkim{ 156199492Sjkim bpf_bin_stream stream; 157153151Sjkim struct bpf_insn *ins; 158199721Sjkim int flags, fret, fpkt, fmem, fjmp, fadk; 159199721Sjkim int save_esp; 160153151Sjkim u_int i, pass; 161153151Sjkim 162153151Sjkim /* 163199615Sjkim * NOTE: Do not modify the name of this variable, as it's used by 164153151Sjkim * the macros to emit code. 165153151Sjkim */ 166153151Sjkim emit_func emitm; 167153151Sjkim 168199721Sjkim flags = bpf_jit_optimize(prog, nins); 169199721Sjkim fret = (flags & BPF_JIT_FRET) != 0; 170199721Sjkim fpkt = (flags & BPF_JIT_FPKT) != 0; 171199721Sjkim fmem = (flags & BPF_JIT_FMEM) != 0; 172199721Sjkim fjmp = (flags & BPF_JIT_FJMP) != 0; 173199721Sjkim fadk = (flags & BPF_JIT_FADK) != 0; 174199721Sjkim save_esp = (fpkt || fmem || fadk); /* Stack is used. */ 175199721Sjkim 176199721Sjkim if (fret) 177199721Sjkim nins = 1; 178199721Sjkim 179199619Sjkim memset(&stream, 0, sizeof(stream)); 180199619Sjkim 181199615Sjkim /* Allocate the reference table for the jumps. */ 182199721Sjkim if (fjmp) { 183181846Sjkim#ifdef _KERNEL 184199619Sjkim stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, 185199619Sjkim M_NOWAIT | M_ZERO); 186181846Sjkim#else 187199721Sjkim stream.refs = calloc(nins + 1, sizeof(u_int)); 188181846Sjkim#endif 189199619Sjkim if (stream.refs == NULL) 190199619Sjkim return (NULL); 191199619Sjkim } 192153151Sjkim 193153151Sjkim /* 194199615Sjkim * The first pass will emit the lengths of the instructions 195199615Sjkim * to create the reference table. 196153151Sjkim */ 197153151Sjkim emitm = emit_length; 198153151Sjkim 199199615Sjkim for (pass = 0; pass < 2; pass++) { 200153151Sjkim ins = prog; 201153151Sjkim 202199615Sjkim /* Create the procedure header. */ 203199721Sjkim if (save_esp) { 204199619Sjkim PUSH(EBP); 205199619Sjkim MOVrd(ESP, EBP); 206199619Sjkim } 207199721Sjkim if (fmem) 208199619Sjkim SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP); 209199721Sjkim if (save_esp) 210199721Sjkim PUSH(ESI); 211199721Sjkim if (fpkt) { 212199619Sjkim PUSH(EDI); 213199619Sjkim PUSH(EBX); 214199619Sjkim MOVodd(8, EBP, EBX); 215199619Sjkim MOVodd(16, EBP, EDI); 216199619Sjkim } 217153151Sjkim 218153151Sjkim for (i = 0; i < nins; i++) { 219153151Sjkim stream.bpf_pc++; 220153151Sjkim 221153151Sjkim switch (ins->code) { 222153151Sjkim default: 223181846Sjkim#ifdef _KERNEL 224181648Sjkim return (NULL); 225181846Sjkim#else 226181846Sjkim abort(); 227181846Sjkim#endif 228153151Sjkim 229153151Sjkim case BPF_RET|BPF_K: 230179968Sjkim MOVid(ins->k, EAX); 231199721Sjkim if (save_esp) { 232199721Sjkim if (fpkt) { 233199721Sjkim POP(EBX); 234199721Sjkim POP(EDI); 235199721Sjkim } 236199619Sjkim POP(ESI); 237199619Sjkim LEAVE(); 238199619Sjkim } 239199619Sjkim RET(); 240153151Sjkim break; 241153151Sjkim 242153151Sjkim case BPF_RET|BPF_A: 243199721Sjkim if (save_esp) { 244199721Sjkim if (fpkt) { 245199721Sjkim POP(EBX); 246199721Sjkim POP(EDI); 247199721Sjkim } 248199619Sjkim POP(ESI); 249199619Sjkim LEAVE(); 250199619Sjkim } 251199619Sjkim RET(); 252153151Sjkim break; 253153151Sjkim 254153151Sjkim case BPF_LD|BPF_W|BPF_ABS: 255181853Sjkim MOVid(ins->k, ESI); 256181853Sjkim CMPrd(EDI, ESI); 257181853Sjkim JAb(12); 258181853Sjkim MOVrd(EDI, ECX); 259181853Sjkim SUBrd(ESI, ECX); 260181853Sjkim CMPid(sizeof(int32_t), ECX); 261181853Sjkim JAEb(7); 262179978Sjkim ZEROrd(EAX); 263153151Sjkim POP(EBX); 264199721Sjkim POP(EDI); 265153151Sjkim POP(ESI); 266199619Sjkim LEAVE(); 267199619Sjkim RET(); 268179968Sjkim MOVobd(EBX, ESI, EAX); 269153151Sjkim BSWAP(EAX); 270153151Sjkim break; 271153151Sjkim 272153151Sjkim case BPF_LD|BPF_H|BPF_ABS: 273179978Sjkim ZEROrd(EAX); 274181853Sjkim MOVid(ins->k, ESI); 275181853Sjkim CMPrd(EDI, ESI); 276181853Sjkim JAb(12); 277181853Sjkim MOVrd(EDI, ECX); 278181853Sjkim SUBrd(ESI, ECX); 279181853Sjkim CMPid(sizeof(int16_t), ECX); 280181853Sjkim JAEb(5); 281153151Sjkim POP(EBX); 282199721Sjkim POP(EDI); 283153151Sjkim POP(ESI); 284199619Sjkim LEAVE(); 285199619Sjkim RET(); 286179968Sjkim MOVobw(EBX, ESI, AX); 287153151Sjkim SWAP_AX(); 288153151Sjkim break; 289153151Sjkim 290153151Sjkim case BPF_LD|BPF_B|BPF_ABS: 291179978Sjkim ZEROrd(EAX); 292181853Sjkim MOVid(ins->k, ESI); 293181853Sjkim CMPrd(EDI, ESI); 294181853Sjkim JBb(5); 295153151Sjkim POP(EBX); 296199721Sjkim POP(EDI); 297153151Sjkim POP(ESI); 298199619Sjkim LEAVE(); 299199619Sjkim RET(); 300181853Sjkim MOVobb(EBX, ESI, AL); 301153151Sjkim break; 302153151Sjkim 303153151Sjkim case BPF_LD|BPF_W|BPF_LEN: 304199721Sjkim if (save_esp) 305199721Sjkim MOVodd(12, EBP, EAX); 306199721Sjkim else { 307199721Sjkim MOVrd(ESP, ECX); 308199721Sjkim MOVodd(12, ECX, EAX); 309199721Sjkim } 310153151Sjkim break; 311153151Sjkim 312153151Sjkim case BPF_LDX|BPF_W|BPF_LEN: 313199721Sjkim if (save_esp) 314199721Sjkim MOVodd(12, EBP, EDX); 315199721Sjkim else { 316199721Sjkim MOVrd(ESP, ECX); 317199721Sjkim MOVodd(12, ECX, EDX); 318199721Sjkim } 319153151Sjkim break; 320153151Sjkim 321153151Sjkim case BPF_LD|BPF_W|BPF_IND: 322181853Sjkim CMPrd(EDI, EDX); 323181853Sjkim JAb(27); 324181853Sjkim MOVid(ins->k, ESI); 325181853Sjkim MOVrd(EDI, ECX); 326181853Sjkim SUBrd(EDX, ECX); 327181853Sjkim CMPrd(ESI, ECX); 328181853Sjkim JBb(14); 329181853Sjkim ADDrd(EDX, ESI); 330181853Sjkim MOVrd(EDI, ECX); 331181853Sjkim SUBrd(ESI, ECX); 332181853Sjkim CMPid(sizeof(int32_t), ECX); 333181853Sjkim JAEb(7); 334179978Sjkim ZEROrd(EAX); 335153151Sjkim POP(EBX); 336199721Sjkim POP(EDI); 337153151Sjkim POP(ESI); 338199619Sjkim LEAVE(); 339199619Sjkim RET(); 340179968Sjkim MOVobd(EBX, ESI, EAX); 341153151Sjkim BSWAP(EAX); 342153151Sjkim break; 343153151Sjkim 344153151Sjkim case BPF_LD|BPF_H|BPF_IND: 345179978Sjkim ZEROrd(EAX); 346181853Sjkim CMPrd(EDI, EDX); 347181853Sjkim JAb(27); 348181853Sjkim MOVid(ins->k, ESI); 349181853Sjkim MOVrd(EDI, ECX); 350181853Sjkim SUBrd(EDX, ECX); 351181853Sjkim CMPrd(ESI, ECX); 352181853Sjkim JBb(14); 353181853Sjkim ADDrd(EDX, ESI); 354181853Sjkim MOVrd(EDI, ECX); 355181853Sjkim SUBrd(ESI, ECX); 356181853Sjkim CMPid(sizeof(int16_t), ECX); 357181853Sjkim JAEb(5); 358153151Sjkim POP(EBX); 359199721Sjkim POP(EDI); 360153151Sjkim POP(ESI); 361199619Sjkim LEAVE(); 362199619Sjkim RET(); 363179968Sjkim MOVobw(EBX, ESI, AX); 364153151Sjkim SWAP_AX(); 365153151Sjkim break; 366153151Sjkim 367153151Sjkim case BPF_LD|BPF_B|BPF_IND: 368179978Sjkim ZEROrd(EAX); 369181853Sjkim CMPrd(EDI, EDX); 370181853Sjkim JAEb(13); 371181853Sjkim MOVid(ins->k, ESI); 372181853Sjkim MOVrd(EDI, ECX); 373181853Sjkim SUBrd(EDX, ECX); 374181853Sjkim CMPrd(ESI, ECX); 375181853Sjkim JAb(5); 376153151Sjkim POP(EBX); 377199721Sjkim POP(EDI); 378153151Sjkim POP(ESI); 379199619Sjkim LEAVE(); 380199619Sjkim RET(); 381181853Sjkim ADDrd(EDX, ESI); 382181853Sjkim MOVobb(EBX, ESI, AL); 383153151Sjkim break; 384153151Sjkim 385153151Sjkim case BPF_LDX|BPF_MSH|BPF_B: 386181853Sjkim MOVid(ins->k, ESI); 387181853Sjkim CMPrd(EDI, ESI); 388181853Sjkim JBb(7); 389179978Sjkim ZEROrd(EAX); 390153151Sjkim POP(EBX); 391199721Sjkim POP(EDI); 392153151Sjkim POP(ESI); 393199619Sjkim LEAVE(); 394199619Sjkim RET(); 395179978Sjkim ZEROrd(EDX); 396181853Sjkim MOVobb(EBX, ESI, DL); 397181648Sjkim ANDib(0x0f, DL); 398179968Sjkim SHLib(2, EDX); 399153151Sjkim break; 400153151Sjkim 401153151Sjkim case BPF_LD|BPF_IMM: 402179968Sjkim MOVid(ins->k, EAX); 403153151Sjkim break; 404153151Sjkim 405153151Sjkim case BPF_LDX|BPF_IMM: 406179968Sjkim MOVid(ins->k, EDX); 407153151Sjkim break; 408153151Sjkim 409153151Sjkim case BPF_LD|BPF_MEM: 410199603Sjkim MOVrd(EBP, ECX); 411199603Sjkim MOVid(((int)ins->k - BPF_MEMWORDS) * 412199603Sjkim sizeof(uint32_t), ESI); 413179968Sjkim MOVobd(ECX, ESI, EAX); 414153151Sjkim break; 415153151Sjkim 416153151Sjkim case BPF_LDX|BPF_MEM: 417199603Sjkim MOVrd(EBP, ECX); 418199603Sjkim MOVid(((int)ins->k - BPF_MEMWORDS) * 419199603Sjkim sizeof(uint32_t), ESI); 420179968Sjkim MOVobd(ECX, ESI, EDX); 421153151Sjkim break; 422153151Sjkim 423153151Sjkim case BPF_ST: 424153151Sjkim /* 425153151Sjkim * XXX this command and the following could 426153151Sjkim * be optimized if the previous instruction 427153151Sjkim * was already of this type 428153151Sjkim */ 429199603Sjkim MOVrd(EBP, ECX); 430199603Sjkim MOVid(((int)ins->k - BPF_MEMWORDS) * 431199603Sjkim sizeof(uint32_t), ESI); 432179968Sjkim MOVomd(EAX, ECX, ESI); 433153151Sjkim break; 434153151Sjkim 435153151Sjkim case BPF_STX: 436199603Sjkim MOVrd(EBP, ECX); 437199603Sjkim MOVid(((int)ins->k - BPF_MEMWORDS) * 438199603Sjkim sizeof(uint32_t), ESI); 439179968Sjkim MOVomd(EDX, ECX, ESI); 440153151Sjkim break; 441153151Sjkim 442153151Sjkim case BPF_JMP|BPF_JA: 443207081Sjkim JUMP(ins->k); 444153151Sjkim break; 445153151Sjkim 446153151Sjkim case BPF_JMP|BPF_JGT|BPF_K: 447207081Sjkim if (ins->jt == ins->jf) { 448207081Sjkim JUMP(ins->jt); 449181697Sjkim break; 450207081Sjkim } 451179968Sjkim CMPid(ins->k, EAX); 452181697Sjkim JCC(JA, JBE); 453153151Sjkim break; 454153151Sjkim 455153151Sjkim case BPF_JMP|BPF_JGE|BPF_K: 456207081Sjkim if (ins->jt == ins->jf) { 457207081Sjkim JUMP(ins->jt); 458181697Sjkim break; 459207081Sjkim } 460179968Sjkim CMPid(ins->k, EAX); 461181697Sjkim JCC(JAE, JB); 462153151Sjkim break; 463153151Sjkim 464153151Sjkim case BPF_JMP|BPF_JEQ|BPF_K: 465207081Sjkim if (ins->jt == ins->jf) { 466207081Sjkim JUMP(ins->jt); 467181697Sjkim break; 468207081Sjkim } 469179968Sjkim CMPid(ins->k, EAX); 470181697Sjkim JCC(JE, JNE); 471153151Sjkim break; 472153151Sjkim 473153151Sjkim case BPF_JMP|BPF_JSET|BPF_K: 474207081Sjkim if (ins->jt == ins->jf) { 475207081Sjkim JUMP(ins->jt); 476181697Sjkim break; 477207081Sjkim } 478181697Sjkim TESTid(ins->k, EAX); 479181697Sjkim JCC(JNE, JE); 480153151Sjkim break; 481153151Sjkim 482153151Sjkim case BPF_JMP|BPF_JGT|BPF_X: 483207081Sjkim if (ins->jt == ins->jf) { 484207081Sjkim JUMP(ins->jt); 485181697Sjkim break; 486207081Sjkim } 487179968Sjkim CMPrd(EDX, EAX); 488181697Sjkim JCC(JA, JBE); 489153151Sjkim break; 490153151Sjkim 491153151Sjkim case BPF_JMP|BPF_JGE|BPF_X: 492207081Sjkim if (ins->jt == ins->jf) { 493207081Sjkim JUMP(ins->jt); 494181697Sjkim break; 495207081Sjkim } 496179968Sjkim CMPrd(EDX, EAX); 497181697Sjkim JCC(JAE, JB); 498153151Sjkim break; 499153151Sjkim 500153151Sjkim case BPF_JMP|BPF_JEQ|BPF_X: 501207081Sjkim if (ins->jt == ins->jf) { 502207081Sjkim JUMP(ins->jt); 503181697Sjkim break; 504207081Sjkim } 505179968Sjkim CMPrd(EDX, EAX); 506181697Sjkim JCC(JE, JNE); 507153151Sjkim break; 508153151Sjkim 509153151Sjkim case BPF_JMP|BPF_JSET|BPF_X: 510207081Sjkim if (ins->jt == ins->jf) { 511207081Sjkim JUMP(ins->jt); 512181697Sjkim break; 513207081Sjkim } 514181697Sjkim TESTrd(EDX, EAX); 515181697Sjkim JCC(JNE, JE); 516153151Sjkim break; 517153151Sjkim 518153151Sjkim case BPF_ALU|BPF_ADD|BPF_X: 519179968Sjkim ADDrd(EDX, EAX); 520153151Sjkim break; 521153151Sjkim 522153151Sjkim case BPF_ALU|BPF_SUB|BPF_X: 523179968Sjkim SUBrd(EDX, EAX); 524153151Sjkim break; 525153151Sjkim 526153151Sjkim case BPF_ALU|BPF_MUL|BPF_X: 527179968Sjkim MOVrd(EDX, ECX); 528179968Sjkim MULrd(EDX); 529153151Sjkim MOVrd(ECX, EDX); 530153151Sjkim break; 531153151Sjkim 532153151Sjkim case BPF_ALU|BPF_DIV|BPF_X: 533181697Sjkim TESTrd(EDX, EDX); 534199721Sjkim if (save_esp) { 535199721Sjkim if (fpkt) { 536199721Sjkim JNEb(7); 537199721Sjkim ZEROrd(EAX); 538199721Sjkim POP(EBX); 539199721Sjkim POP(EDI); 540199721Sjkim } else { 541199721Sjkim JNEb(5); 542199721Sjkim ZEROrd(EAX); 543199721Sjkim } 544199721Sjkim POP(ESI); 545199721Sjkim LEAVE(); 546199721Sjkim } else { 547199721Sjkim JNEb(3); 548199721Sjkim ZEROrd(EAX); 549199721Sjkim } 550199619Sjkim RET(); 551179968Sjkim MOVrd(EDX, ECX); 552179978Sjkim ZEROrd(EDX); 553153151Sjkim DIVrd(ECX); 554179968Sjkim MOVrd(ECX, EDX); 555153151Sjkim break; 556153151Sjkim 557153151Sjkim case BPF_ALU|BPF_AND|BPF_X: 558179968Sjkim ANDrd(EDX, EAX); 559153151Sjkim break; 560153151Sjkim 561153151Sjkim case BPF_ALU|BPF_OR|BPF_X: 562179968Sjkim ORrd(EDX, EAX); 563153151Sjkim break; 564153151Sjkim 565153151Sjkim case BPF_ALU|BPF_LSH|BPF_X: 566179968Sjkim MOVrd(EDX, ECX); 567153151Sjkim SHL_CLrb(EAX); 568153151Sjkim break; 569153151Sjkim 570153151Sjkim case BPF_ALU|BPF_RSH|BPF_X: 571179968Sjkim MOVrd(EDX, ECX); 572153151Sjkim SHR_CLrb(EAX); 573153151Sjkim break; 574153151Sjkim 575153151Sjkim case BPF_ALU|BPF_ADD|BPF_K: 576153151Sjkim ADD_EAXi(ins->k); 577153151Sjkim break; 578153151Sjkim 579153151Sjkim case BPF_ALU|BPF_SUB|BPF_K: 580153151Sjkim SUB_EAXi(ins->k); 581153151Sjkim break; 582153151Sjkim 583153151Sjkim case BPF_ALU|BPF_MUL|BPF_K: 584179968Sjkim MOVrd(EDX, ECX); 585179968Sjkim MOVid(ins->k, EDX); 586179968Sjkim MULrd(EDX); 587153151Sjkim MOVrd(ECX, EDX); 588153151Sjkim break; 589153151Sjkim 590153151Sjkim case BPF_ALU|BPF_DIV|BPF_K: 591179968Sjkim MOVrd(EDX, ECX); 592179978Sjkim ZEROrd(EDX); 593179968Sjkim MOVid(ins->k, ESI); 594153151Sjkim DIVrd(ESI); 595179968Sjkim MOVrd(ECX, EDX); 596153151Sjkim break; 597153151Sjkim 598153151Sjkim case BPF_ALU|BPF_AND|BPF_K: 599179968Sjkim ANDid(ins->k, EAX); 600153151Sjkim break; 601153151Sjkim 602153151Sjkim case BPF_ALU|BPF_OR|BPF_K: 603179968Sjkim ORid(ins->k, EAX); 604153151Sjkim break; 605153151Sjkim 606153151Sjkim case BPF_ALU|BPF_LSH|BPF_K: 607179968Sjkim SHLib((ins->k) & 0xff, EAX); 608153151Sjkim break; 609153151Sjkim 610153151Sjkim case BPF_ALU|BPF_RSH|BPF_K: 611179968Sjkim SHRib((ins->k) & 0xff, EAX); 612153151Sjkim break; 613153151Sjkim 614153151Sjkim case BPF_ALU|BPF_NEG: 615153151Sjkim NEGd(EAX); 616153151Sjkim break; 617153151Sjkim 618153151Sjkim case BPF_MISC|BPF_TAX: 619179968Sjkim MOVrd(EAX, EDX); 620153151Sjkim break; 621153151Sjkim 622153151Sjkim case BPF_MISC|BPF_TXA: 623179968Sjkim MOVrd(EDX, EAX); 624153151Sjkim break; 625153151Sjkim } 626153151Sjkim ins++; 627153151Sjkim } 628153151Sjkim 629199615Sjkim if (pass > 0) 630199615Sjkim continue; 631153151Sjkim 632199615Sjkim *size = stream.cur_ip; 633181846Sjkim#ifdef _KERNEL 634199615Sjkim stream.ibuf = malloc(*size, M_BPFJIT, M_NOWAIT); 635199492Sjkim if (stream.ibuf == NULL) 636199492Sjkim break; 637181846Sjkim#else 638199615Sjkim stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE, 639199603Sjkim MAP_ANON, -1, 0); 640199492Sjkim if (stream.ibuf == MAP_FAILED) { 641199492Sjkim stream.ibuf = NULL; 642199492Sjkim break; 643199492Sjkim } 644181846Sjkim#endif 645153151Sjkim 646153151Sjkim /* 647199615Sjkim * Modify the reference table to contain the offsets and 648199615Sjkim * not the lengths of the instructions. 649153151Sjkim */ 650199721Sjkim if (fjmp) 651199619Sjkim for (i = 1; i < nins + 1; i++) 652199619Sjkim stream.refs[i] += stream.refs[i - 1]; 653153151Sjkim 654199615Sjkim /* Reset the counters. */ 655153151Sjkim stream.cur_ip = 0; 656153151Sjkim stream.bpf_pc = 0; 657153151Sjkim 658199615Sjkim /* The second pass creates the actual code. */ 659153151Sjkim emitm = emit_code; 660153151Sjkim } 661153151Sjkim 662153151Sjkim /* 663199615Sjkim * The reference table is needed only during compilation, 664199615Sjkim * now we can free it. 665153151Sjkim */ 666199721Sjkim if (fjmp) 667181846Sjkim#ifdef _KERNEL 668199619Sjkim free(stream.refs, M_BPFJIT); 669181846Sjkim#else 670199619Sjkim free(stream.refs); 671199619Sjkim#endif 672199619Sjkim 673199619Sjkim#ifndef _KERNEL 674199615Sjkim if (stream.ibuf != NULL && 675199615Sjkim mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) { 676199615Sjkim munmap(stream.ibuf, *size); 677199615Sjkim stream.ibuf = NULL; 678199615Sjkim } 679181846Sjkim#endif 680153151Sjkim 681181648Sjkim return ((bpf_filter_func)stream.ibuf); 682153151Sjkim} 683