x86/net/bpf_jit_comp.c

606 static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog,
657 	if (bpf_prog->aux->exception_boundary) {
662 		if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena))
673 	      offsetof(struct bpf_prog, bpf_func));
688 static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog,
718 	if (bpf_prog->aux->exception_boundary) {
723 		if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena))
739 static void bpf_tail_call_direct_fixup(struct bpf_prog *prog)
743 	struct bpf_prog *target;
1269 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,
1272 	bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
1273 	struct bpf_insn *insn = bpf_prog->insnsi;
1275 	int insn_cnt = bpf_prog->len;
1285 	arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena);
1286 	user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena);
1294 	emit_prologue(&prog, bpf_prog->aux->stack_depth,
1295 		      bpf_prog_was_classic(bpf_prog), tail_call_reachable,
1296 		      bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb);
1300 	if (bpf_prog->aux->exception_boundary) {
1767 				if (!bpf_prog->aux->extable)
1770 				if (excnt >= bpf_prog->aux->num_exentries) {
1774 				ex = &bpf_prog->aux->extable[excnt++];
1870 				if (!bpf_prog->aux->extable)
1873 				if (excnt >= bpf_prog->aux->num_exentries) {
1877 				ex = &bpf_prog->aux->extable[excnt++];
1972 				RESTORE_TAIL_CALL_CNT(bpf_prog->aux->stack_depth);
1985 				emit_bpf_tail_call_direct(bpf_prog,
1986 							  &bpf_prog->aux->poke_tab[imm32 - 1],
1989 							  bpf_prog->aux->stack_depth,
1992 				emit_bpf_tail_call_indirect(bpf_prog,
1995 							    bpf_prog->aux->stack_depth,
2246 			if (bpf_prog->aux->exception_boundary) {
2263 			 * junk in bpf_prog.
2296 	if (image && excnt != bpf_prog->aux->num_exentries) {
2485 	struct bpf_prog *p = l->link.prog;
3158 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
3162 	struct bpf_prog *tmp, *orig_prog = prog;
3362 void bpf_jit_free(struct bpf_prog *prog)
3415 			       struct bpf_prog *new, struct bpf_prog *old)