1153151Sjkim/*- 2181648Sjkim * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) 3199492Sjkim * Copyright (C) 2005-2009 Jung-uk Kim <jkim@FreeBSD.org> 4153151Sjkim * All rights reserved. 5153151Sjkim * 6153151Sjkim * Redistribution and use in source and binary forms, with or without 7153151Sjkim * modification, are permitted provided that the following conditions 8153151Sjkim * are met: 9153151Sjkim * 10153151Sjkim * 1. Redistributions of source code must retain the above copyright 11153151Sjkim * notice, this list of conditions and the following disclaimer. 12153151Sjkim * 2. Redistributions in binary form must reproduce the above copyright 13153151Sjkim * notice, this list of conditions and the following disclaimer in the 14153151Sjkim * documentation and/or other materials provided with the distribution. 15153151Sjkim * 3. Neither the name of the Politecnico di Torino nor the names of its 16153151Sjkim * contributors may be used to endorse or promote products derived from 17153151Sjkim * this software without specific prior written permission. 18153151Sjkim * 19153151Sjkim * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20153151Sjkim * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21153151Sjkim * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22153151Sjkim * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23153151Sjkim * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24153151Sjkim * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25153151Sjkim * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26182173Sjkim * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27153151Sjkim * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28153151Sjkim * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29153151Sjkim * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30153151Sjkim */ 31153151Sjkim 32153151Sjkim#include <sys/cdefs.h> 33153151Sjkim__FBSDID("$FreeBSD$"); 34153151Sjkim 35181846Sjkim#ifdef _KERNEL 36153151Sjkim#include "opt_bpf.h" 37153151Sjkim#include <sys/param.h> 38153151Sjkim#include <sys/systm.h> 39153151Sjkim#include <sys/kernel.h> 40257324Sglebius#include <sys/mbuf.h> 41153151Sjkim#include <sys/socket.h> 42153151Sjkim#include <sys/malloc.h> 43181846Sjkim#include <net/if.h> 44181846Sjkim#else 45181846Sjkim#include <stdlib.h> 46199615Sjkim#include <string.h> 47199492Sjkim#include <sys/mman.h> 48199492Sjkim#include <sys/param.h> 49181846Sjkim#endif 50153151Sjkim 51181846Sjkim#include <sys/types.h> 52181846Sjkim 53153151Sjkim#include <net/bpf.h> 54153151Sjkim#include <net/bpf_jitter.h> 55153151Sjkim 56153151Sjkim#include <i386/i386/bpf_jit_machdep.h> 57153151Sjkim 58199603Sjkimbpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); 59153151Sjkim 60153151Sjkim/* 61199619Sjkim * Emit routine to update the jump table. 62153151Sjkim */ 63153151Sjkimstatic void 64181846Sjkimemit_length(bpf_bin_stream *stream, __unused u_int value, u_int len) 65153151Sjkim{ 66153151Sjkim 67199619Sjkim if (stream->refs != NULL) 68199619Sjkim (stream->refs)[stream->bpf_pc] += len; 69153151Sjkim stream->cur_ip += len; 70153151Sjkim} 71153151Sjkim 72153151Sjkim/* 73199619Sjkim * Emit routine to output the actual binary code. 74153151Sjkim */ 75153151Sjkimstatic void 76153151Sjkimemit_code(bpf_bin_stream *stream, u_int value, u_int len) 77153151Sjkim{ 78153151Sjkim 79153151Sjkim switch (len) { 80153151Sjkim case 1: 81153151Sjkim stream->ibuf[stream->cur_ip] = (u_char)value; 82153151Sjkim stream->cur_ip++; 83153151Sjkim break; 84153151Sjkim 85153151Sjkim case 2: 86153151Sjkim *((u_short *)(stream->ibuf + stream->cur_ip)) = (u_short)value; 87153151Sjkim stream->cur_ip += 2; 88153151Sjkim break; 89153151Sjkim 90153151Sjkim case 4: 91153151Sjkim *((u_int *)(stream->ibuf + stream->cur_ip)) = value; 92153151Sjkim stream->cur_ip += 4; 93153151Sjkim break; 94153151Sjkim } 95153151Sjkim 96153151Sjkim return; 97153151Sjkim} 98153151Sjkim 99153151Sjkim/* 100199619Sjkim * Scan the filter program and find possible optimization. 101153151Sjkim */ 102199619Sjkimstatic int 103199619Sjkimbpf_jit_optimize(struct bpf_insn *prog, u_int nins) 104199619Sjkim{ 105199619Sjkim int flags; 106199619Sjkim u_int i; 107199619Sjkim 108199619Sjkim /* Do we return immediately? */ 109199619Sjkim if (BPF_CLASS(prog[0].code) == BPF_RET) 110199721Sjkim return (BPF_JIT_FRET); 111199619Sjkim 112199619Sjkim for (flags = 0, i = 0; i < nins; i++) { 113199721Sjkim switch (prog[i].code) { 114199721Sjkim case BPF_LD|BPF_W|BPF_ABS: 115199721Sjkim case BPF_LD|BPF_H|BPF_ABS: 116199721Sjkim case BPF_LD|BPF_B|BPF_ABS: 117199721Sjkim case BPF_LD|BPF_W|BPF_IND: 118199721Sjkim case BPF_LD|BPF_H|BPF_IND: 119199721Sjkim case BPF_LD|BPF_B|BPF_IND: 120199721Sjkim case BPF_LDX|BPF_MSH|BPF_B: 121199721Sjkim flags |= BPF_JIT_FPKT; 122199721Sjkim break; 123199721Sjkim case BPF_LD|BPF_MEM: 124199721Sjkim case BPF_LDX|BPF_MEM: 125199721Sjkim case BPF_ST: 126199721Sjkim case BPF_STX: 127199721Sjkim flags |= BPF_JIT_FMEM; 128199721Sjkim break; 129199721Sjkim case BPF_JMP|BPF_JA: 130199721Sjkim case BPF_JMP|BPF_JGT|BPF_K: 131199721Sjkim case BPF_JMP|BPF_JGE|BPF_K: 132199721Sjkim case BPF_JMP|BPF_JEQ|BPF_K: 133199721Sjkim case BPF_JMP|BPF_JSET|BPF_K: 134199721Sjkim case BPF_JMP|BPF_JGT|BPF_X: 135199721Sjkim case BPF_JMP|BPF_JGE|BPF_X: 136199721Sjkim case BPF_JMP|BPF_JEQ|BPF_X: 137199721Sjkim case BPF_JMP|BPF_JSET|BPF_X: 138199721Sjkim flags |= BPF_JIT_FJMP; 139199721Sjkim break; 140199721Sjkim case BPF_ALU|BPF_DIV|BPF_K: 141199721Sjkim flags |= BPF_JIT_FADK; 142199721Sjkim break; 143199721Sjkim } 144199619Sjkim if (flags == BPF_JIT_FLAG_ALL) 145199619Sjkim break; 146199619Sjkim } 147199619Sjkim 148199619Sjkim return (flags); 149199619Sjkim} 150199619Sjkim 151199619Sjkim/* 152199619Sjkim * Function that does the real stuff. 153199619Sjkim */ 154153151Sjkimbpf_filter_func 155199603Sjkimbpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) 156153151Sjkim{ 157199492Sjkim bpf_bin_stream stream; 158153151Sjkim struct bpf_insn *ins; 159199721Sjkim int flags, fret, fpkt, fmem, fjmp, fadk; 160199721Sjkim int save_esp; 161153151Sjkim u_int i, pass; 162153151Sjkim 163153151Sjkim /* 164199615Sjkim * NOTE: Do not modify the name of this variable, as it's used by 165153151Sjkim * the macros to emit code. 166153151Sjkim */ 167153151Sjkim emit_func emitm; 168153151Sjkim 169199721Sjkim flags = bpf_jit_optimize(prog, nins); 170199721Sjkim fret = (flags & BPF_JIT_FRET) != 0; 171199721Sjkim fpkt = (flags & BPF_JIT_FPKT) != 0; 172199721Sjkim fmem = (flags & BPF_JIT_FMEM) != 0; 173199721Sjkim fjmp = (flags & BPF_JIT_FJMP) != 0; 174199721Sjkim fadk = (flags & BPF_JIT_FADK) != 0; 175199721Sjkim save_esp = (fpkt || fmem || fadk); /* Stack is used. */ 176199721Sjkim 177199721Sjkim if (fret) 178199721Sjkim nins = 1; 179199721Sjkim 180199619Sjkim memset(&stream, 0, sizeof(stream)); 181199619Sjkim 182199615Sjkim /* Allocate the reference table for the jumps. */ 183199721Sjkim if (fjmp) { 184181846Sjkim#ifdef _KERNEL 185199619Sjkim stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, 186199619Sjkim M_NOWAIT | M_ZERO); 187181846Sjkim#else 188199721Sjkim stream.refs = calloc(nins + 1, sizeof(u_int)); 189181846Sjkim#endif 190199619Sjkim if (stream.refs == NULL) 191199619Sjkim return (NULL); 192199619Sjkim } 193153151Sjkim 194153151Sjkim /* 195199615Sjkim * The first pass will emit the lengths of the instructions 196199615Sjkim * to create the reference table. 197153151Sjkim */ 198153151Sjkim emitm = emit_length; 199153151Sjkim 200199615Sjkim for (pass = 0; pass < 2; pass++) { 201153151Sjkim ins = prog; 202153151Sjkim 203199615Sjkim /* Create the procedure header. */ 204199721Sjkim if (save_esp) { 205199619Sjkim PUSH(EBP); 206199619Sjkim MOVrd(ESP, EBP); 207199619Sjkim } 208199721Sjkim if (fmem) 209199619Sjkim SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP); 210199721Sjkim if (save_esp) 211199721Sjkim PUSH(ESI); 212199721Sjkim if (fpkt) { 213199619Sjkim PUSH(EDI); 214199619Sjkim PUSH(EBX); 215199619Sjkim MOVodd(8, EBP, EBX); 216199619Sjkim MOVodd(16, EBP, EDI); 217199619Sjkim } 218153151Sjkim 219153151Sjkim for (i = 0; i < nins; i++) { 220153151Sjkim stream.bpf_pc++; 221153151Sjkim 222153151Sjkim switch (ins->code) { 223153151Sjkim default: 224181846Sjkim#ifdef _KERNEL 225181648Sjkim return (NULL); 226181846Sjkim#else 227181846Sjkim abort(); 228181846Sjkim#endif 229153151Sjkim 230153151Sjkim case BPF_RET|BPF_K: 231179968Sjkim MOVid(ins->k, EAX); 232199721Sjkim if (save_esp) { 233199721Sjkim if (fpkt) { 234199721Sjkim POP(EBX); 235199721Sjkim POP(EDI); 236199721Sjkim } 237199619Sjkim POP(ESI); 238199619Sjkim LEAVE(); 239199619Sjkim } 240199619Sjkim RET(); 241153151Sjkim break; 242153151Sjkim 243153151Sjkim case BPF_RET|BPF_A: 244199721Sjkim if (save_esp) { 245199721Sjkim if (fpkt) { 246199721Sjkim POP(EBX); 247199721Sjkim POP(EDI); 248199721Sjkim } 249199619Sjkim POP(ESI); 250199619Sjkim LEAVE(); 251199619Sjkim } 252199619Sjkim RET(); 253153151Sjkim break; 254153151Sjkim 255153151Sjkim case BPF_LD|BPF_W|BPF_ABS: 256181853Sjkim MOVid(ins->k, ESI); 257181853Sjkim CMPrd(EDI, ESI); 258181853Sjkim JAb(12); 259181853Sjkim MOVrd(EDI, ECX); 260181853Sjkim SUBrd(ESI, ECX); 261181853Sjkim CMPid(sizeof(int32_t), ECX); 262181853Sjkim JAEb(7); 263179978Sjkim ZEROrd(EAX); 264153151Sjkim POP(EBX); 265199721Sjkim POP(EDI); 266153151Sjkim POP(ESI); 267199619Sjkim LEAVE(); 268199619Sjkim RET(); 269179968Sjkim MOVobd(EBX, ESI, EAX); 270153151Sjkim BSWAP(EAX); 271153151Sjkim break; 272153151Sjkim 273153151Sjkim case BPF_LD|BPF_H|BPF_ABS: 274179978Sjkim ZEROrd(EAX); 275181853Sjkim MOVid(ins->k, ESI); 276181853Sjkim CMPrd(EDI, ESI); 277181853Sjkim JAb(12); 278181853Sjkim MOVrd(EDI, ECX); 279181853Sjkim SUBrd(ESI, ECX); 280181853Sjkim CMPid(sizeof(int16_t), ECX); 281181853Sjkim JAEb(5); 282153151Sjkim POP(EBX); 283199721Sjkim POP(EDI); 284153151Sjkim POP(ESI); 285199619Sjkim LEAVE(); 286199619Sjkim RET(); 287179968Sjkim MOVobw(EBX, ESI, AX); 288153151Sjkim SWAP_AX(); 289153151Sjkim break; 290153151Sjkim 291153151Sjkim case BPF_LD|BPF_B|BPF_ABS: 292179978Sjkim ZEROrd(EAX); 293181853Sjkim MOVid(ins->k, ESI); 294181853Sjkim CMPrd(EDI, ESI); 295181853Sjkim JBb(5); 296153151Sjkim POP(EBX); 297199721Sjkim POP(EDI); 298153151Sjkim POP(ESI); 299199619Sjkim LEAVE(); 300199619Sjkim RET(); 301181853Sjkim MOVobb(EBX, ESI, AL); 302153151Sjkim break; 303153151Sjkim 304153151Sjkim case BPF_LD|BPF_W|BPF_LEN: 305199721Sjkim if (save_esp) 306199721Sjkim MOVodd(12, EBP, EAX); 307199721Sjkim else { 308199721Sjkim MOVrd(ESP, ECX); 309199721Sjkim MOVodd(12, ECX, EAX); 310199721Sjkim } 311153151Sjkim break; 312153151Sjkim 313153151Sjkim case BPF_LDX|BPF_W|BPF_LEN: 314199721Sjkim if (save_esp) 315199721Sjkim MOVodd(12, EBP, EDX); 316199721Sjkim else { 317199721Sjkim MOVrd(ESP, ECX); 318199721Sjkim MOVodd(12, ECX, EDX); 319199721Sjkim } 320153151Sjkim break; 321153151Sjkim 322153151Sjkim case BPF_LD|BPF_W|BPF_IND: 323181853Sjkim CMPrd(EDI, EDX); 324181853Sjkim JAb(27); 325181853Sjkim MOVid(ins->k, ESI); 326181853Sjkim MOVrd(EDI, ECX); 327181853Sjkim SUBrd(EDX, ECX); 328181853Sjkim CMPrd(ESI, ECX); 329181853Sjkim JBb(14); 330181853Sjkim ADDrd(EDX, ESI); 331181853Sjkim MOVrd(EDI, ECX); 332181853Sjkim SUBrd(ESI, ECX); 333181853Sjkim CMPid(sizeof(int32_t), ECX); 334181853Sjkim JAEb(7); 335179978Sjkim ZEROrd(EAX); 336153151Sjkim POP(EBX); 337199721Sjkim POP(EDI); 338153151Sjkim POP(ESI); 339199619Sjkim LEAVE(); 340199619Sjkim RET(); 341179968Sjkim MOVobd(EBX, ESI, EAX); 342153151Sjkim BSWAP(EAX); 343153151Sjkim break; 344153151Sjkim 345153151Sjkim case BPF_LD|BPF_H|BPF_IND: 346179978Sjkim ZEROrd(EAX); 347181853Sjkim CMPrd(EDI, EDX); 348181853Sjkim JAb(27); 349181853Sjkim MOVid(ins->k, ESI); 350181853Sjkim MOVrd(EDI, ECX); 351181853Sjkim SUBrd(EDX, ECX); 352181853Sjkim CMPrd(ESI, ECX); 353181853Sjkim JBb(14); 354181853Sjkim ADDrd(EDX, ESI); 355181853Sjkim MOVrd(EDI, ECX); 356181853Sjkim SUBrd(ESI, ECX); 357181853Sjkim CMPid(sizeof(int16_t), ECX); 358181853Sjkim JAEb(5); 359153151Sjkim POP(EBX); 360199721Sjkim POP(EDI); 361153151Sjkim POP(ESI); 362199619Sjkim LEAVE(); 363199619Sjkim RET(); 364179968Sjkim MOVobw(EBX, ESI, AX); 365153151Sjkim SWAP_AX(); 366153151Sjkim break; 367153151Sjkim 368153151Sjkim case BPF_LD|BPF_B|BPF_IND: 369179978Sjkim ZEROrd(EAX); 370181853Sjkim CMPrd(EDI, EDX); 371181853Sjkim JAEb(13); 372181853Sjkim MOVid(ins->k, ESI); 373181853Sjkim MOVrd(EDI, ECX); 374181853Sjkim SUBrd(EDX, ECX); 375181853Sjkim CMPrd(ESI, ECX); 376181853Sjkim JAb(5); 377153151Sjkim POP(EBX); 378199721Sjkim POP(EDI); 379153151Sjkim POP(ESI); 380199619Sjkim LEAVE(); 381199619Sjkim RET(); 382181853Sjkim ADDrd(EDX, ESI); 383181853Sjkim MOVobb(EBX, ESI, AL); 384153151Sjkim break; 385153151Sjkim 386153151Sjkim case BPF_LDX|BPF_MSH|BPF_B: 387181853Sjkim MOVid(ins->k, ESI); 388181853Sjkim CMPrd(EDI, ESI); 389181853Sjkim JBb(7); 390179978Sjkim ZEROrd(EAX); 391153151Sjkim POP(EBX); 392199721Sjkim POP(EDI); 393153151Sjkim POP(ESI); 394199619Sjkim LEAVE(); 395199619Sjkim RET(); 396179978Sjkim ZEROrd(EDX); 397181853Sjkim MOVobb(EBX, ESI, DL); 398181648Sjkim ANDib(0x0f, DL); 399179968Sjkim SHLib(2, EDX); 400153151Sjkim break; 401153151Sjkim 402153151Sjkim case BPF_LD|BPF_IMM: 403179968Sjkim MOVid(ins->k, EAX); 404153151Sjkim break; 405153151Sjkim 406153151Sjkim case BPF_LDX|BPF_IMM: 407179968Sjkim MOVid(ins->k, EDX); 408153151Sjkim break; 409153151Sjkim 410153151Sjkim case BPF_LD|BPF_MEM: 411199603Sjkim MOVrd(EBP, ECX); 412199603Sjkim MOVid(((int)ins->k - BPF_MEMWORDS) * 413199603Sjkim sizeof(uint32_t), ESI); 414179968Sjkim MOVobd(ECX, ESI, EAX); 415153151Sjkim break; 416153151Sjkim 417153151Sjkim case BPF_LDX|BPF_MEM: 418199603Sjkim MOVrd(EBP, ECX); 419199603Sjkim MOVid(((int)ins->k - BPF_MEMWORDS) * 420199603Sjkim sizeof(uint32_t), ESI); 421179968Sjkim MOVobd(ECX, ESI, EDX); 422153151Sjkim break; 423153151Sjkim 424153151Sjkim case BPF_ST: 425153151Sjkim /* 426153151Sjkim * XXX this command and the following could 427153151Sjkim * be optimized if the previous instruction 428153151Sjkim * was already of this type 429153151Sjkim */ 430199603Sjkim MOVrd(EBP, ECX); 431199603Sjkim MOVid(((int)ins->k - BPF_MEMWORDS) * 432199603Sjkim sizeof(uint32_t), ESI); 433179968Sjkim MOVomd(EAX, ECX, ESI); 434153151Sjkim break; 435153151Sjkim 436153151Sjkim case BPF_STX: 437199603Sjkim MOVrd(EBP, ECX); 438199603Sjkim MOVid(((int)ins->k - BPF_MEMWORDS) * 439199603Sjkim sizeof(uint32_t), ESI); 440179968Sjkim MOVomd(EDX, ECX, ESI); 441153151Sjkim break; 442153151Sjkim 443153151Sjkim case BPF_JMP|BPF_JA: 444207081Sjkim JUMP(ins->k); 445153151Sjkim break; 446153151Sjkim 447153151Sjkim case BPF_JMP|BPF_JGT|BPF_K: 448207081Sjkim if (ins->jt == ins->jf) { 449207081Sjkim JUMP(ins->jt); 450181697Sjkim break; 451207081Sjkim } 452179968Sjkim CMPid(ins->k, EAX); 453181697Sjkim JCC(JA, JBE); 454153151Sjkim break; 455153151Sjkim 456153151Sjkim case BPF_JMP|BPF_JGE|BPF_K: 457207081Sjkim if (ins->jt == ins->jf) { 458207081Sjkim JUMP(ins->jt); 459181697Sjkim break; 460207081Sjkim } 461179968Sjkim CMPid(ins->k, EAX); 462181697Sjkim JCC(JAE, JB); 463153151Sjkim break; 464153151Sjkim 465153151Sjkim case BPF_JMP|BPF_JEQ|BPF_K: 466207081Sjkim if (ins->jt == ins->jf) { 467207081Sjkim JUMP(ins->jt); 468181697Sjkim break; 469207081Sjkim } 470179968Sjkim CMPid(ins->k, EAX); 471181697Sjkim JCC(JE, JNE); 472153151Sjkim break; 473153151Sjkim 474153151Sjkim case BPF_JMP|BPF_JSET|BPF_K: 475207081Sjkim if (ins->jt == ins->jf) { 476207081Sjkim JUMP(ins->jt); 477181697Sjkim break; 478207081Sjkim } 479181697Sjkim TESTid(ins->k, EAX); 480181697Sjkim JCC(JNE, JE); 481153151Sjkim break; 482153151Sjkim 483153151Sjkim case BPF_JMP|BPF_JGT|BPF_X: 484207081Sjkim if (ins->jt == ins->jf) { 485207081Sjkim JUMP(ins->jt); 486181697Sjkim break; 487207081Sjkim } 488179968Sjkim CMPrd(EDX, EAX); 489181697Sjkim JCC(JA, JBE); 490153151Sjkim break; 491153151Sjkim 492153151Sjkim case BPF_JMP|BPF_JGE|BPF_X: 493207081Sjkim if (ins->jt == ins->jf) { 494207081Sjkim JUMP(ins->jt); 495181697Sjkim break; 496207081Sjkim } 497179968Sjkim CMPrd(EDX, EAX); 498181697Sjkim JCC(JAE, JB); 499153151Sjkim break; 500153151Sjkim 501153151Sjkim case BPF_JMP|BPF_JEQ|BPF_X: 502207081Sjkim if (ins->jt == ins->jf) { 503207081Sjkim JUMP(ins->jt); 504181697Sjkim break; 505207081Sjkim } 506179968Sjkim CMPrd(EDX, EAX); 507181697Sjkim JCC(JE, JNE); 508153151Sjkim break; 509153151Sjkim 510153151Sjkim case BPF_JMP|BPF_JSET|BPF_X: 511207081Sjkim if (ins->jt == ins->jf) { 512207081Sjkim JUMP(ins->jt); 513181697Sjkim break; 514207081Sjkim } 515181697Sjkim TESTrd(EDX, EAX); 516181697Sjkim JCC(JNE, JE); 517153151Sjkim break; 518153151Sjkim 519153151Sjkim case BPF_ALU|BPF_ADD|BPF_X: 520179968Sjkim ADDrd(EDX, EAX); 521153151Sjkim break; 522153151Sjkim 523153151Sjkim case BPF_ALU|BPF_SUB|BPF_X: 524179968Sjkim SUBrd(EDX, EAX); 525153151Sjkim break; 526153151Sjkim 527153151Sjkim case BPF_ALU|BPF_MUL|BPF_X: 528179968Sjkim MOVrd(EDX, ECX); 529179968Sjkim MULrd(EDX); 530153151Sjkim MOVrd(ECX, EDX); 531153151Sjkim break; 532153151Sjkim 533153151Sjkim case BPF_ALU|BPF_DIV|BPF_X: 534181697Sjkim TESTrd(EDX, EDX); 535199721Sjkim if (save_esp) { 536199721Sjkim if (fpkt) { 537199721Sjkim JNEb(7); 538199721Sjkim ZEROrd(EAX); 539199721Sjkim POP(EBX); 540199721Sjkim POP(EDI); 541199721Sjkim } else { 542199721Sjkim JNEb(5); 543199721Sjkim ZEROrd(EAX); 544199721Sjkim } 545199721Sjkim POP(ESI); 546199721Sjkim LEAVE(); 547199721Sjkim } else { 548199721Sjkim JNEb(3); 549199721Sjkim ZEROrd(EAX); 550199721Sjkim } 551199619Sjkim RET(); 552179968Sjkim MOVrd(EDX, ECX); 553179978Sjkim ZEROrd(EDX); 554153151Sjkim DIVrd(ECX); 555179968Sjkim MOVrd(ECX, EDX); 556153151Sjkim break; 557153151Sjkim 558153151Sjkim case BPF_ALU|BPF_AND|BPF_X: 559179968Sjkim ANDrd(EDX, EAX); 560153151Sjkim break; 561153151Sjkim 562153151Sjkim case BPF_ALU|BPF_OR|BPF_X: 563179968Sjkim ORrd(EDX, EAX); 564153151Sjkim break; 565153151Sjkim 566153151Sjkim case BPF_ALU|BPF_LSH|BPF_X: 567179968Sjkim MOVrd(EDX, ECX); 568153151Sjkim SHL_CLrb(EAX); 569153151Sjkim break; 570153151Sjkim 571153151Sjkim case BPF_ALU|BPF_RSH|BPF_X: 572179968Sjkim MOVrd(EDX, ECX); 573153151Sjkim SHR_CLrb(EAX); 574153151Sjkim break; 575153151Sjkim 576153151Sjkim case BPF_ALU|BPF_ADD|BPF_K: 577153151Sjkim ADD_EAXi(ins->k); 578153151Sjkim break; 579153151Sjkim 580153151Sjkim case BPF_ALU|BPF_SUB|BPF_K: 581153151Sjkim SUB_EAXi(ins->k); 582153151Sjkim break; 583153151Sjkim 584153151Sjkim case BPF_ALU|BPF_MUL|BPF_K: 585179968Sjkim MOVrd(EDX, ECX); 586179968Sjkim MOVid(ins->k, EDX); 587179968Sjkim MULrd(EDX); 588153151Sjkim MOVrd(ECX, EDX); 589153151Sjkim break; 590153151Sjkim 591153151Sjkim case BPF_ALU|BPF_DIV|BPF_K: 592179968Sjkim MOVrd(EDX, ECX); 593179978Sjkim ZEROrd(EDX); 594179968Sjkim MOVid(ins->k, ESI); 595153151Sjkim DIVrd(ESI); 596179968Sjkim MOVrd(ECX, EDX); 597153151Sjkim break; 598153151Sjkim 599153151Sjkim case BPF_ALU|BPF_AND|BPF_K: 600179968Sjkim ANDid(ins->k, EAX); 601153151Sjkim break; 602153151Sjkim 603153151Sjkim case BPF_ALU|BPF_OR|BPF_K: 604179968Sjkim ORid(ins->k, EAX); 605153151Sjkim break; 606153151Sjkim 607153151Sjkim case BPF_ALU|BPF_LSH|BPF_K: 608179968Sjkim SHLib((ins->k) & 0xff, EAX); 609153151Sjkim break; 610153151Sjkim 611153151Sjkim case BPF_ALU|BPF_RSH|BPF_K: 612179968Sjkim SHRib((ins->k) & 0xff, EAX); 613153151Sjkim break; 614153151Sjkim 615153151Sjkim case BPF_ALU|BPF_NEG: 616153151Sjkim NEGd(EAX); 617153151Sjkim break; 618153151Sjkim 619153151Sjkim case BPF_MISC|BPF_TAX: 620179968Sjkim MOVrd(EAX, EDX); 621153151Sjkim break; 622153151Sjkim 623153151Sjkim case BPF_MISC|BPF_TXA: 624179968Sjkim MOVrd(EDX, EAX); 625153151Sjkim break; 626153151Sjkim } 627153151Sjkim ins++; 628153151Sjkim } 629153151Sjkim 630199615Sjkim if (pass > 0) 631199615Sjkim continue; 632153151Sjkim 633199615Sjkim *size = stream.cur_ip; 634181846Sjkim#ifdef _KERNEL 635199615Sjkim stream.ibuf = malloc(*size, M_BPFJIT, M_NOWAIT); 636199492Sjkim if (stream.ibuf == NULL) 637199492Sjkim break; 638181846Sjkim#else 639199615Sjkim stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE, 640199603Sjkim MAP_ANON, -1, 0); 641199492Sjkim if (stream.ibuf == MAP_FAILED) { 642199492Sjkim stream.ibuf = NULL; 643199492Sjkim break; 644199492Sjkim } 645181846Sjkim#endif 646153151Sjkim 647153151Sjkim /* 648199615Sjkim * Modify the reference table to contain the offsets and 649199615Sjkim * not the lengths of the instructions. 650153151Sjkim */ 651199721Sjkim if (fjmp) 652199619Sjkim for (i = 1; i < nins + 1; i++) 653199619Sjkim stream.refs[i] += stream.refs[i - 1]; 654153151Sjkim 655199615Sjkim /* Reset the counters. */ 656153151Sjkim stream.cur_ip = 0; 657153151Sjkim stream.bpf_pc = 0; 658153151Sjkim 659199615Sjkim /* The second pass creates the actual code. */ 660153151Sjkim emitm = emit_code; 661153151Sjkim } 662153151Sjkim 663153151Sjkim /* 664199615Sjkim * The reference table is needed only during compilation, 665199615Sjkim * now we can free it. 666153151Sjkim */ 667199721Sjkim if (fjmp) 668181846Sjkim#ifdef _KERNEL 669199619Sjkim free(stream.refs, M_BPFJIT); 670181846Sjkim#else 671199619Sjkim free(stream.refs); 672199619Sjkim#endif 673199619Sjkim 674199619Sjkim#ifndef _KERNEL 675199615Sjkim if (stream.ibuf != NULL && 676199615Sjkim mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) { 677199615Sjkim munmap(stream.ibuf, *size); 678199615Sjkim stream.ibuf = NULL; 679199615Sjkim } 680181846Sjkim#endif 681153151Sjkim 682181648Sjkim return ((bpf_filter_func)stream.ibuf); 683153151Sjkim} 684