bpf_jit_machdep.c revision 199492
1153151Sjkim/*- 2181648Sjkim * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) 3199492Sjkim * Copyright (C) 2005-2009 Jung-uk Kim <jkim@FreeBSD.org> 4153151Sjkim * All rights reserved. 5153151Sjkim * 6153151Sjkim * Redistribution and use in source and binary forms, with or without 7153151Sjkim * modification, are permitted provided that the following conditions 8153151Sjkim * are met: 9153151Sjkim * 10153151Sjkim * 1. Redistributions of source code must retain the above copyright 11153151Sjkim * notice, this list of conditions and the following disclaimer. 12153151Sjkim * 2. Redistributions in binary form must reproduce the above copyright 13153151Sjkim * notice, this list of conditions and the following disclaimer in the 14153151Sjkim * documentation and/or other materials provided with the distribution. 15153151Sjkim * 3. Neither the name of the Politecnico di Torino nor the names of its 16153151Sjkim * contributors may be used to endorse or promote products derived from 17153151Sjkim * this software without specific prior written permission. 18153151Sjkim * 19153151Sjkim * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20153151Sjkim * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21153151Sjkim * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22153151Sjkim * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23153151Sjkim * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24153151Sjkim * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25153151Sjkim * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26182173Sjkim * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27153151Sjkim * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28153151Sjkim * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29153151Sjkim * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30153151Sjkim */ 31153151Sjkim 32153151Sjkim#include <sys/cdefs.h> 33153151Sjkim__FBSDID("$FreeBSD: head/sys/i386/i386/bpf_jit_machdep.c 199492 2009-11-18 19:26:17Z jkim $"); 34153151Sjkim 35181846Sjkim#ifdef _KERNEL 36153151Sjkim#include "opt_bpf.h" 37153151Sjkim#include <sys/param.h> 38153151Sjkim#include <sys/systm.h> 39153151Sjkim#include <sys/kernel.h> 40153151Sjkim#include <sys/socket.h> 41153151Sjkim#include <sys/malloc.h> 42181846Sjkim#include <net/if.h> 43181846Sjkim#else 44181846Sjkim#include <stdlib.h> 45199492Sjkim#include <sys/mman.h> 46199492Sjkim#include <sys/param.h> 47181846Sjkim#endif 48153151Sjkim 49181846Sjkim#include <sys/types.h> 50181846Sjkim 51153151Sjkim#include <net/bpf.h> 52153151Sjkim#include <net/bpf_jitter.h> 53153151Sjkim 54153151Sjkim#include <i386/i386/bpf_jit_machdep.h> 55153151Sjkim 56153151Sjkimbpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, int *); 57153151Sjkim 58153151Sjkim/* 59153151Sjkim * emit routine to update the jump table 60153151Sjkim */ 61153151Sjkimstatic void 62181846Sjkimemit_length(bpf_bin_stream *stream, __unused u_int value, u_int len) 63153151Sjkim{ 64153151Sjkim 65153151Sjkim (stream->refs)[stream->bpf_pc] += len; 66153151Sjkim stream->cur_ip += len; 67153151Sjkim} 68153151Sjkim 69153151Sjkim/* 70153151Sjkim * emit routine to output the actual binary code 71153151Sjkim */ 72153151Sjkimstatic void 73153151Sjkimemit_code(bpf_bin_stream *stream, u_int value, u_int len) 74153151Sjkim{ 75153151Sjkim 76153151Sjkim switch (len) { 77153151Sjkim case 1: 78153151Sjkim stream->ibuf[stream->cur_ip] = (u_char)value; 79153151Sjkim stream->cur_ip++; 80153151Sjkim break; 81153151Sjkim 82153151Sjkim case 2: 83153151Sjkim *((u_short *)(stream->ibuf + stream->cur_ip)) = (u_short)value; 84153151Sjkim stream->cur_ip += 2; 85153151Sjkim break; 86153151Sjkim 87153151Sjkim case 4: 88153151Sjkim *((u_int *)(stream->ibuf + stream->cur_ip)) = value; 89153151Sjkim stream->cur_ip += 4; 90153151Sjkim break; 91153151Sjkim } 92153151Sjkim 93153151Sjkim return; 94153151Sjkim} 95153151Sjkim 96153151Sjkim/* 97153151Sjkim * Function that does the real stuff 98153151Sjkim */ 99153151Sjkimbpf_filter_func 100153151Sjkimbpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem) 101153151Sjkim{ 102199492Sjkim bpf_bin_stream stream; 103153151Sjkim struct bpf_insn *ins; 104153151Sjkim u_int i, pass; 105153151Sjkim 106153151Sjkim /* 107153151Sjkim * NOTE: do not modify the name of this variable, as it's used by 108153151Sjkim * the macros to emit code. 109153151Sjkim */ 110153151Sjkim emit_func emitm; 111153151Sjkim 112153151Sjkim /* Allocate the reference table for the jumps */ 113181846Sjkim#ifdef _KERNEL 114153151Sjkim stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int), 115153157Sjkim M_BPFJIT, M_NOWAIT); 116181846Sjkim#else 117181846Sjkim stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int)); 118181846Sjkim#endif 119153151Sjkim if (stream.refs == NULL) 120181648Sjkim return (NULL); 121153151Sjkim 122153151Sjkim /* Reset the reference table */ 123153151Sjkim for (i = 0; i < nins + 1; i++) 124153151Sjkim stream.refs[i] = 0; 125153151Sjkim 126153151Sjkim stream.cur_ip = 0; 127153151Sjkim stream.bpf_pc = 0; 128153151Sjkim 129153151Sjkim /* 130153151Sjkim * the first pass will emit the lengths of the instructions 131153151Sjkim * to create the reference table 132153151Sjkim */ 133153151Sjkim emitm = emit_length; 134153151Sjkim 135153151Sjkim pass = 0; 136153151Sjkim for (;;) { 137153151Sjkim ins = prog; 138153151Sjkim 139153151Sjkim /* create the procedure header */ 140153151Sjkim PUSH(EBP); 141179968Sjkim MOVrd(ESP, EBP); 142153151Sjkim PUSH(EDI); 143153151Sjkim PUSH(ESI); 144153151Sjkim PUSH(EBX); 145179968Sjkim MOVodd(8, EBP, EBX); 146181645Sjkim MOVodd(16, EBP, EDI); 147153151Sjkim 148153151Sjkim for (i = 0; i < nins; i++) { 149153151Sjkim stream.bpf_pc++; 150153151Sjkim 151153151Sjkim switch (ins->code) { 152153151Sjkim default: 153181846Sjkim#ifdef _KERNEL 154181648Sjkim return (NULL); 155181846Sjkim#else 156181846Sjkim abort(); 157181846Sjkim#endif 158153151Sjkim 159153151Sjkim case BPF_RET|BPF_K: 160179968Sjkim MOVid(ins->k, EAX); 161153151Sjkim POP(EBX); 162153151Sjkim POP(ESI); 163153151Sjkim POP(EDI); 164153151Sjkim LEAVE_RET(); 165153151Sjkim break; 166153151Sjkim 167153151Sjkim case BPF_RET|BPF_A: 168153151Sjkim POP(EBX); 169153151Sjkim POP(ESI); 170153151Sjkim POP(EDI); 171153151Sjkim LEAVE_RET(); 172153151Sjkim break; 173153151Sjkim 174153151Sjkim case BPF_LD|BPF_W|BPF_ABS: 175181853Sjkim MOVid(ins->k, ESI); 176181853Sjkim CMPrd(EDI, ESI); 177181853Sjkim JAb(12); 178181853Sjkim MOVrd(EDI, ECX); 179181853Sjkim SUBrd(ESI, ECX); 180181853Sjkim CMPid(sizeof(int32_t), ECX); 181181853Sjkim JAEb(7); 182179978Sjkim ZEROrd(EAX); 183153151Sjkim POP(EBX); 184153151Sjkim POP(ESI); 185153151Sjkim POP(EDI); 186153151Sjkim LEAVE_RET(); 187179968Sjkim MOVobd(EBX, ESI, EAX); 188153151Sjkim BSWAP(EAX); 189153151Sjkim break; 190153151Sjkim 191153151Sjkim case BPF_LD|BPF_H|BPF_ABS: 192179978Sjkim ZEROrd(EAX); 193181853Sjkim MOVid(ins->k, ESI); 194181853Sjkim CMPrd(EDI, ESI); 195181853Sjkim JAb(12); 196181853Sjkim MOVrd(EDI, ECX); 197181853Sjkim SUBrd(ESI, ECX); 198181853Sjkim CMPid(sizeof(int16_t), ECX); 199181853Sjkim JAEb(5); 200153151Sjkim POP(EBX); 201153151Sjkim POP(ESI); 202153151Sjkim POP(EDI); 203153151Sjkim LEAVE_RET(); 204179968Sjkim MOVobw(EBX, ESI, AX); 205153151Sjkim SWAP_AX(); 206153151Sjkim break; 207153151Sjkim 208153151Sjkim case BPF_LD|BPF_B|BPF_ABS: 209179978Sjkim ZEROrd(EAX); 210181853Sjkim MOVid(ins->k, ESI); 211181853Sjkim CMPrd(EDI, ESI); 212181853Sjkim JBb(5); 213153151Sjkim POP(EBX); 214153151Sjkim POP(ESI); 215153151Sjkim POP(EDI); 216153151Sjkim LEAVE_RET(); 217181853Sjkim MOVobb(EBX, ESI, AL); 218153151Sjkim break; 219153151Sjkim 220153151Sjkim case BPF_LD|BPF_W|BPF_LEN: 221181648Sjkim MOVodd(12, EBP, EAX); 222153151Sjkim break; 223153151Sjkim 224153151Sjkim case BPF_LDX|BPF_W|BPF_LEN: 225181648Sjkim MOVodd(12, EBP, EDX); 226153151Sjkim break; 227153151Sjkim 228153151Sjkim case BPF_LD|BPF_W|BPF_IND: 229181853Sjkim CMPrd(EDI, EDX); 230181853Sjkim JAb(27); 231181853Sjkim MOVid(ins->k, ESI); 232181853Sjkim MOVrd(EDI, ECX); 233181853Sjkim SUBrd(EDX, ECX); 234181853Sjkim CMPrd(ESI, ECX); 235181853Sjkim JBb(14); 236181853Sjkim ADDrd(EDX, ESI); 237181853Sjkim MOVrd(EDI, ECX); 238181853Sjkim SUBrd(ESI, ECX); 239181853Sjkim CMPid(sizeof(int32_t), ECX); 240181853Sjkim JAEb(7); 241179978Sjkim ZEROrd(EAX); 242153151Sjkim POP(EBX); 243153151Sjkim POP(ESI); 244153151Sjkim POP(EDI); 245153151Sjkim LEAVE_RET(); 246179968Sjkim MOVobd(EBX, ESI, EAX); 247153151Sjkim BSWAP(EAX); 248153151Sjkim break; 249153151Sjkim 250153151Sjkim case BPF_LD|BPF_H|BPF_IND: 251179978Sjkim ZEROrd(EAX); 252181853Sjkim CMPrd(EDI, EDX); 253181853Sjkim JAb(27); 254181853Sjkim MOVid(ins->k, ESI); 255181853Sjkim MOVrd(EDI, ECX); 256181853Sjkim SUBrd(EDX, ECX); 257181853Sjkim CMPrd(ESI, ECX); 258181853Sjkim JBb(14); 259181853Sjkim ADDrd(EDX, ESI); 260181853Sjkim MOVrd(EDI, ECX); 261181853Sjkim SUBrd(ESI, ECX); 262181853Sjkim CMPid(sizeof(int16_t), ECX); 263181853Sjkim JAEb(5); 264153151Sjkim POP(EBX); 265153151Sjkim POP(ESI); 266153151Sjkim POP(EDI); 267153151Sjkim LEAVE_RET(); 268179968Sjkim MOVobw(EBX, ESI, AX); 269153151Sjkim SWAP_AX(); 270153151Sjkim break; 271153151Sjkim 272153151Sjkim case BPF_LD|BPF_B|BPF_IND: 273179978Sjkim ZEROrd(EAX); 274181853Sjkim CMPrd(EDI, EDX); 275181853Sjkim JAEb(13); 276181853Sjkim MOVid(ins->k, ESI); 277181853Sjkim MOVrd(EDI, ECX); 278181853Sjkim SUBrd(EDX, ECX); 279181853Sjkim CMPrd(ESI, ECX); 280181853Sjkim JAb(5); 281153151Sjkim POP(EBX); 282153151Sjkim POP(ESI); 283153151Sjkim POP(EDI); 284153151Sjkim LEAVE_RET(); 285181853Sjkim ADDrd(EDX, ESI); 286181853Sjkim MOVobb(EBX, ESI, AL); 287153151Sjkim break; 288153151Sjkim 289153151Sjkim case BPF_LDX|BPF_MSH|BPF_B: 290181853Sjkim MOVid(ins->k, ESI); 291181853Sjkim CMPrd(EDI, ESI); 292181853Sjkim JBb(7); 293179978Sjkim ZEROrd(EAX); 294153151Sjkim POP(EBX); 295153151Sjkim POP(ESI); 296153151Sjkim POP(EDI); 297153151Sjkim LEAVE_RET(); 298179978Sjkim ZEROrd(EDX); 299181853Sjkim MOVobb(EBX, ESI, DL); 300181648Sjkim ANDib(0x0f, DL); 301179968Sjkim SHLib(2, EDX); 302153151Sjkim break; 303153151Sjkim 304153151Sjkim case BPF_LD|BPF_IMM: 305179968Sjkim MOVid(ins->k, EAX); 306153151Sjkim break; 307153151Sjkim 308153151Sjkim case BPF_LDX|BPF_IMM: 309179968Sjkim MOVid(ins->k, EDX); 310153151Sjkim break; 311153151Sjkim 312153151Sjkim case BPF_LD|BPF_MEM: 313179968Sjkim MOVid((uintptr_t)mem, ECX); 314179968Sjkim MOVid(ins->k * 4, ESI); 315179968Sjkim MOVobd(ECX, ESI, EAX); 316153151Sjkim break; 317153151Sjkim 318153151Sjkim case BPF_LDX|BPF_MEM: 319179968Sjkim MOVid((uintptr_t)mem, ECX); 320179968Sjkim MOVid(ins->k * 4, ESI); 321179968Sjkim MOVobd(ECX, ESI, EDX); 322153151Sjkim break; 323153151Sjkim 324153151Sjkim case BPF_ST: 325153151Sjkim /* 326153151Sjkim * XXX this command and the following could 327153151Sjkim * be optimized if the previous instruction 328153151Sjkim * was already of this type 329153151Sjkim */ 330179968Sjkim MOVid((uintptr_t)mem, ECX); 331179968Sjkim MOVid(ins->k * 4, ESI); 332179968Sjkim MOVomd(EAX, ECX, ESI); 333153151Sjkim break; 334153151Sjkim 335153151Sjkim case BPF_STX: 336179968Sjkim MOVid((uintptr_t)mem, ECX); 337179968Sjkim MOVid(ins->k * 4, ESI); 338179968Sjkim MOVomd(EDX, ECX, ESI); 339153151Sjkim break; 340153151Sjkim 341153151Sjkim case BPF_JMP|BPF_JA: 342153151Sjkim JMP(stream.refs[stream.bpf_pc + ins->k] - 343153151Sjkim stream.refs[stream.bpf_pc]); 344153151Sjkim break; 345153151Sjkim 346153151Sjkim case BPF_JMP|BPF_JGT|BPF_K: 347181697Sjkim if (ins->jt == 0 && ins->jf == 0) 348181697Sjkim break; 349179968Sjkim CMPid(ins->k, EAX); 350181697Sjkim JCC(JA, JBE); 351153151Sjkim break; 352153151Sjkim 353153151Sjkim case BPF_JMP|BPF_JGE|BPF_K: 354181697Sjkim if (ins->jt == 0 && ins->jf == 0) 355181697Sjkim break; 356179968Sjkim CMPid(ins->k, EAX); 357181697Sjkim JCC(JAE, JB); 358153151Sjkim break; 359153151Sjkim 360153151Sjkim case BPF_JMP|BPF_JEQ|BPF_K: 361181697Sjkim if (ins->jt == 0 && ins->jf == 0) 362181697Sjkim break; 363179968Sjkim CMPid(ins->k, EAX); 364181697Sjkim JCC(JE, JNE); 365153151Sjkim break; 366153151Sjkim 367153151Sjkim case BPF_JMP|BPF_JSET|BPF_K: 368181697Sjkim if (ins->jt == 0 && ins->jf == 0) 369181697Sjkim break; 370181697Sjkim TESTid(ins->k, EAX); 371181697Sjkim JCC(JNE, JE); 372153151Sjkim break; 373153151Sjkim 374153151Sjkim case BPF_JMP|BPF_JGT|BPF_X: 375181697Sjkim if (ins->jt == 0 && ins->jf == 0) 376181697Sjkim break; 377179968Sjkim CMPrd(EDX, EAX); 378181697Sjkim JCC(JA, JBE); 379153151Sjkim break; 380153151Sjkim 381153151Sjkim case BPF_JMP|BPF_JGE|BPF_X: 382181697Sjkim if (ins->jt == 0 && ins->jf == 0) 383181697Sjkim break; 384179968Sjkim CMPrd(EDX, EAX); 385181697Sjkim JCC(JAE, JB); 386153151Sjkim break; 387153151Sjkim 388153151Sjkim case BPF_JMP|BPF_JEQ|BPF_X: 389181697Sjkim if (ins->jt == 0 && ins->jf == 0) 390181697Sjkim break; 391179968Sjkim CMPrd(EDX, EAX); 392181697Sjkim JCC(JE, JNE); 393153151Sjkim break; 394153151Sjkim 395153151Sjkim case BPF_JMP|BPF_JSET|BPF_X: 396181697Sjkim if (ins->jt == 0 && ins->jf == 0) 397181697Sjkim break; 398181697Sjkim TESTrd(EDX, EAX); 399181697Sjkim JCC(JNE, JE); 400153151Sjkim break; 401153151Sjkim 402153151Sjkim case BPF_ALU|BPF_ADD|BPF_X: 403179968Sjkim ADDrd(EDX, EAX); 404153151Sjkim break; 405153151Sjkim 406153151Sjkim case BPF_ALU|BPF_SUB|BPF_X: 407179968Sjkim SUBrd(EDX, EAX); 408153151Sjkim break; 409153151Sjkim 410153151Sjkim case BPF_ALU|BPF_MUL|BPF_X: 411179968Sjkim MOVrd(EDX, ECX); 412179968Sjkim MULrd(EDX); 413153151Sjkim MOVrd(ECX, EDX); 414153151Sjkim break; 415153151Sjkim 416153151Sjkim case BPF_ALU|BPF_DIV|BPF_X: 417181697Sjkim TESTrd(EDX, EDX); 418153151Sjkim JNEb(7); 419179978Sjkim ZEROrd(EAX); 420153151Sjkim POP(EBX); 421153151Sjkim POP(ESI); 422153151Sjkim POP(EDI); 423153151Sjkim LEAVE_RET(); 424179968Sjkim MOVrd(EDX, ECX); 425179978Sjkim ZEROrd(EDX); 426153151Sjkim DIVrd(ECX); 427179968Sjkim MOVrd(ECX, EDX); 428153151Sjkim break; 429153151Sjkim 430153151Sjkim case BPF_ALU|BPF_AND|BPF_X: 431179968Sjkim ANDrd(EDX, EAX); 432153151Sjkim break; 433153151Sjkim 434153151Sjkim case BPF_ALU|BPF_OR|BPF_X: 435179968Sjkim ORrd(EDX, EAX); 436153151Sjkim break; 437153151Sjkim 438153151Sjkim case BPF_ALU|BPF_LSH|BPF_X: 439179968Sjkim MOVrd(EDX, ECX); 440153151Sjkim SHL_CLrb(EAX); 441153151Sjkim break; 442153151Sjkim 443153151Sjkim case BPF_ALU|BPF_RSH|BPF_X: 444179968Sjkim MOVrd(EDX, ECX); 445153151Sjkim SHR_CLrb(EAX); 446153151Sjkim break; 447153151Sjkim 448153151Sjkim case BPF_ALU|BPF_ADD|BPF_K: 449153151Sjkim ADD_EAXi(ins->k); 450153151Sjkim break; 451153151Sjkim 452153151Sjkim case BPF_ALU|BPF_SUB|BPF_K: 453153151Sjkim SUB_EAXi(ins->k); 454153151Sjkim break; 455153151Sjkim 456153151Sjkim case BPF_ALU|BPF_MUL|BPF_K: 457179968Sjkim MOVrd(EDX, ECX); 458179968Sjkim MOVid(ins->k, EDX); 459179968Sjkim MULrd(EDX); 460153151Sjkim MOVrd(ECX, EDX); 461153151Sjkim break; 462153151Sjkim 463153151Sjkim case BPF_ALU|BPF_DIV|BPF_K: 464179968Sjkim MOVrd(EDX, ECX); 465179978Sjkim ZEROrd(EDX); 466179968Sjkim MOVid(ins->k, ESI); 467153151Sjkim DIVrd(ESI); 468179968Sjkim MOVrd(ECX, EDX); 469153151Sjkim break; 470153151Sjkim 471153151Sjkim case BPF_ALU|BPF_AND|BPF_K: 472179968Sjkim ANDid(ins->k, EAX); 473153151Sjkim break; 474153151Sjkim 475153151Sjkim case BPF_ALU|BPF_OR|BPF_K: 476179968Sjkim ORid(ins->k, EAX); 477153151Sjkim break; 478153151Sjkim 479153151Sjkim case BPF_ALU|BPF_LSH|BPF_K: 480179968Sjkim SHLib((ins->k) & 0xff, EAX); 481153151Sjkim break; 482153151Sjkim 483153151Sjkim case BPF_ALU|BPF_RSH|BPF_K: 484179968Sjkim SHRib((ins->k) & 0xff, EAX); 485153151Sjkim break; 486153151Sjkim 487153151Sjkim case BPF_ALU|BPF_NEG: 488153151Sjkim NEGd(EAX); 489153151Sjkim break; 490153151Sjkim 491153151Sjkim case BPF_MISC|BPF_TAX: 492179968Sjkim MOVrd(EAX, EDX); 493153151Sjkim break; 494153151Sjkim 495153151Sjkim case BPF_MISC|BPF_TXA: 496179968Sjkim MOVrd(EDX, EAX); 497153151Sjkim break; 498153151Sjkim } 499153151Sjkim ins++; 500153151Sjkim } 501153151Sjkim 502153151Sjkim pass++; 503199492Sjkim if (pass >= 2) { 504199492Sjkim#ifndef _KERNEL 505199492Sjkim if (mprotect(stream.ibuf, stream.cur_ip, 506199492Sjkim PROT_READ | PROT_EXEC) != 0) { 507199492Sjkim munmap(stream.ibuf, BPF_JIT_MAXSIZE); 508199492Sjkim stream.ibuf = NULL; 509199492Sjkim } 510199492Sjkim#endif 511153151Sjkim break; 512199492Sjkim } 513153151Sjkim 514181846Sjkim#ifdef _KERNEL 515153157Sjkim stream.ibuf = (char *)malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT); 516199492Sjkim if (stream.ibuf == NULL) 517199492Sjkim break; 518181846Sjkim#else 519199492Sjkim if (stream.cur_ip > BPF_JIT_MAXSIZE) { 520199492Sjkim stream.ibuf = NULL; 521199492Sjkim break; 522181846Sjkim } 523199492Sjkim stream.ibuf = (char *)mmap(NULL, BPF_JIT_MAXSIZE, 524199492Sjkim PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); 525199492Sjkim if (stream.ibuf == MAP_FAILED) { 526199492Sjkim stream.ibuf = NULL; 527199492Sjkim break; 528199492Sjkim } 529181846Sjkim#endif 530153151Sjkim 531153151Sjkim /* 532153151Sjkim * modify the reference table to contain the offsets and 533153151Sjkim * not the lengths of the instructions 534153151Sjkim */ 535153151Sjkim for (i = 1; i < nins + 1; i++) 536153151Sjkim stream.refs[i] += stream.refs[i - 1]; 537153151Sjkim 538153151Sjkim /* Reset the counters */ 539153151Sjkim stream.cur_ip = 0; 540153151Sjkim stream.bpf_pc = 0; 541153151Sjkim 542153151Sjkim /* the second pass creates the actual code */ 543153151Sjkim emitm = emit_code; 544153151Sjkim } 545153151Sjkim 546153151Sjkim /* 547153151Sjkim * the reference table is needed only during compilation, 548153151Sjkim * now we can free it 549153151Sjkim */ 550181846Sjkim#ifdef _KERNEL 551153151Sjkim free(stream.refs, M_BPFJIT); 552181846Sjkim#else 553181846Sjkim free(stream.refs); 554181846Sjkim#endif 555153151Sjkim 556181648Sjkim return ((bpf_filter_func)stream.ibuf); 557153151Sjkim} 558