Lines Matching defs:subprog
424 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)
428 return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;
431 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog)
438 info = &env->prog->aux->func_info[subprog];
442 static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog)
444 struct bpf_subprog_info *info = subprog_info(env, subprog);
451 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog)
453 return subprog_info(env, subprog)->is_exception_cb;
2405 int subprog)
2438 subprog /* subprog number within this prog */);
2490 /* determine subprog starts. The end is one before the next starts */
2513 verbose(env, "invalid btf id for main subprog in func_info\n");
2524 verbose(env, "multiple exception callback tags for main subprog\n");
2544 /* Further func_info and subprog checks will also happen
2897 struct bpf_subprog_info *subprog = env->subprog_info;
2930 /* If ex_cb_insn > 0, this means that the main program has a subprog
2946 /* Add a fake 'exit' subprog which could simplify subprog iteration
2949 subprog[env->subprog_cnt].start = insn_cnt;
2953 verbose(env, "func#%d @%d\n", i, subprog[i].start);
2961 struct bpf_subprog_info *subprog = env->subprog_info;
2965 /* now check that all jumps are within the same subprog */
2966 subprog_start = subprog[cur_subprog].start;
2967 subprog_end = subprog[cur_subprog + 1].start;
2974 subprog[cur_subprog].has_tail_call = true;
2977 subprog[cur_subprog].has_ld_abs = true;
2992 /* to avoid fall-through from one subprog into another
2993 * the last insn of the subprog should be either exit
3005 subprog_end = subprog[cur_subprog + 1].start;
3446 verbose(bt->env, "BUG subprog enter from frame %d\n", bt->frame);
3457 verbose(bt->env, "BUG subprog exit from frame 0\n");
3678 int subprog_insn_idx, subprog;
3681 subprog = find_subprog(env, subprog_insn_idx);
3682 if (subprog < 0)
3685 if (subprog_is_global(env, subprog)) {
3687 * extra instructions from subprog; the next
3688 * instruction after call to global subprog
3693 /* r1-r5 are invalidated after subprog call,
3702 /* global subprog always sets R0 */
3706 /* static subprog call instruction, which
3707 * means that we are exiting current subprog,
3722 WARN_ONCE(1, "verifier backtracking bug (subprog leftover stack slots)");
3737 /* exit from callback subprog to callback-calling helper or
3740 * Unlike the subprog call handling above, we shouldn't
3754 /* clear r1-r5 in callback subprog's mask */
3797 /* BPF_EXIT in subprog or callback always returns
3799 * whether the instruction at subseq_idx-1 is subprog
3801 * *subprog* from exit from *callback*. In the former
4223 /* we are at the entry into subprog, which
4243 verbose(env, "BUG backtracking func entry subprog %d reg_mask %x stack_mask %llx\n",
5867 struct bpf_subprog_info *subprog = env->subprog_info;
5875 i = subprog[idx].start;
5896 if (idx && subprog[idx].has_tail_call && depth >= 256) {
5902 depth += round_up_stack_depth(env, subprog[idx].stack_depth);
5909 subprog_end = subprog[idx + 1].start;
5918 if (subprog[idx].is_cb)
5921 if (subprog[ret_prog[c]].is_cb) {
5929 "bpf_throw kfunc (insn %d) cannot be called from callback subprog %d\n",
5948 if (subprog[sidx].is_async_cb) {
5949 if (subprog[sidx].has_tail_call) {
5950 verbose(env, "verifier bug. subprog has tail_call and async cb\n");
5956 if (subprog[sidx].is_exception_cb) {
5964 if (subprog[idx].has_tail_call)
5976 * currently present subprog frames as tail call reachable subprogs;
5982 if (subprog[ret_prog[j]].is_exception_cb) {
5986 subprog[ret_prog[j]].tail_call_reachable = true;
5988 if (subprog[0].tail_call_reachable)
5991 /* end of for() loop means the last insn of the 'subprog'
5996 depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
6023 int start = idx + insn->imm + 1, subprog;
6025 subprog = find_subprog(env, start);
6026 if (subprog < 0) {
6031 return env->subprog_info[subprog].stack_depth;
9297 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,
9330 subprog /* subprog number within this prog */);
9348 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
9352 struct bpf_subprog_info *sub = subprog_info(env, subprog);
9357 ret = btf_prepare_func_args(env, subprog);
9440 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
9451 btf_id = prog->aux->func_info[subprog].type_id;
9455 if (prog->aux->func_info_aux[subprog].unreliable)
9458 err = btf_check_func_arg_match(env, subprog, btf, regs);
9464 prog->aux->func_info_aux[subprog].unreliable = true;
9469 int insn_idx, int subprog,
9477 err = btf_check_subprog_call(env, subprog, caller->regs);
9481 /* set_callee_state is used for direct subprog calls, but we are
9485 env->subprog_info[subprog].is_cb = true;
9502 env->subprog_info[subprog].is_async_cb = true;
9503 async_cb = push_async_cb(env, env->subprog_info[subprog].start,
9504 insn_idx, subprog);
9521 callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);
9525 err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,
9541 int err, subprog, target_insn;
9544 subprog = find_subprog(env, target_insn);
9545 if (subprog < 0) {
9551 err = btf_check_subprog_call(env, subprog, caller->regs);
9554 if (subprog_is_global(env, subprog)) {
9555 const char *sub_name = subprog_name(env, subprog);
9566 subprog, sub_name);
9571 subprog, sub_name);
9572 /* mark global subprog for verifying after main prog */
9573 subprog_aux(env, subprog)->called = true;
9587 err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);
9594 *insn_idx = env->subprog_info[subprog].start - 1;
15680 /* when we exit from subprog, we need to record non-linear history */
16044 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
18490 /* NOTE: fake 'exit' subprog should be updated as well. */
18563 /* move fake 'exit' subprog as well */
18589 /* update fake 'exit' subprog as well */
19113 int i, j, subprog_start, subprog_end = 0, len, subprog;
19130 subprog = find_subprog(env, i + insn->imm + 1);
19131 if (subprog < 0) {
19136 /* temporarily remember subprog id inside insn instead of
19139 insn->off = subprog;
19245 subprog = insn->off;
19246 insn[0].imm = (u32)(long)func[subprog]->bpf_func;
19247 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32;
19252 subprog = insn->off;
19253 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func);
19264 * by using its subprog id, available from the off field of
19305 subprog = find_subprog(env, i + insn->off + 1);
19306 insn->imm = subprog;
19324 * patch it anymore as we're freeing the subprog JIT memory.
19563 verbose(env, "verifier internal error: only one hidden subprog supported\n");
19567 * ones for the hidden subprog. Hence all of the adjustment operations
20234 /* Add ST insn to subprog prologue to init extra stack */
20354 * subprog stack_depth is increased by the size of 3 registers.
20434 static int do_check_common(struct bpf_verifier_env *env, int subprog)
20437 struct bpf_subprog_info *sub = subprog_info(env, subprog);
20460 subprog);
20461 state->first_insn_idx = env->subprog_info[subprog].start;
20465 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
20466 const char *sub_name = subprog_name(env, subprog);
20470 verbose(env, "Validating %s() func#%d...\n", sub_name, subprog);
20471 ret = btf_prepare_func_args(env, subprog);
20475 if (subprog_is_exc_cb(env, subprog)) {
20614 /* We verified new global subprog, it might have called some
20622 /* We can't loop forever as we verify at least one global subprog on
20783 int ret = 0, subprog = -1, i;
20822 subprog = i;
20825 if (subprog == -1) {
20829 if (aux->func && aux->func[subprog]->aux->exception_cb) {
20835 conservative = aux->func_info_aux[subprog].unreliable;
20974 if (subprog == 0)
20977 addr = (long) tgt_prog->aux->func[subprog]->bpf_func;