Lines Matching refs:ctx
35 static void mark_call(struct jit_ctx *ctx)
37 ctx->flags |= SAVE_RA;
40 static void mark_tail_call(struct jit_ctx *ctx)
42 ctx->flags |= SAVE_TCC;
45 static bool seen_call(struct jit_ctx *ctx)
47 return (ctx->flags & SAVE_RA);
50 static bool seen_tail_call(struct jit_ctx *ctx)
52 return (ctx->flags & SAVE_TCC);
55 static u8 tail_call_reg(struct jit_ctx *ctx)
57 if (seen_call(ctx))
89 static void build_prologue(struct jit_ctx *ctx)
93 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
106 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
108 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
111 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
114 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
117 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
120 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
123 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
126 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
129 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
132 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
134 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
137 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
143 if (seen_tail_call(ctx) && seen_call(ctx))
144 move_reg(ctx, TCC_SAVED, REG_TCC);
146 ctx->stack_size = stack_adjust;
149 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
151 int stack_adjust = ctx->stack_size;
155 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
158 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
161 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
164 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
167 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
170 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
173 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
176 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
178 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
182 move_reg(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0]);
184 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, LOONGARCH_GPR_ZERO, 0);
190 emit_insn(ctx, jirl, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, 1);
194 static void build_epilogue(struct jit_ctx *ctx)
196 __build_epilogue(ctx, false);
211 static int emit_bpf_tail_call(struct jit_ctx *ctx)
214 u8 tcc = tail_call_reg(ctx);
220 const int idx0 = ctx->idx;
222 #define cur_offset (ctx->idx - idx0)
226 * a0: &ctx
234 emit_insn(ctx, ldwu, t1, a1, off);
236 if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
243 emit_insn(ctx, addid, REG_TCC, tcc, -1);
244 if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
252 emit_insn(ctx, alsld, t2, a2, a1, 2);
254 emit_insn(ctx, ldd, t2, t2, off);
256 if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
261 emit_insn(ctx, ldd, t3, t2, off);
262 __build_epilogue(ctx, true);
282 static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
294 move_imm(ctx, t1, off, false);
295 emit_insn(ctx, addd, t1, dst, t1);
296 move_reg(ctx, t3, src);
302 emit_insn(ctx, amaddd, t2, t1, src);
304 emit_insn(ctx, amaddw, t2, t1, src);
308 emit_insn(ctx, amandd, t2, t1, src);
310 emit_insn(ctx, amandw, t2, t1, src);
314 emit_insn(ctx, amord, t2, t1, src);
316 emit_insn(ctx, amorw, t2, t1, src);
320 emit_insn(ctx, amxord, t2, t1, src);
322 emit_insn(ctx, amxorw, t2, t1, src);
327 emit_insn(ctx, amaddd, src, t1, t3);
329 emit_insn(ctx, amaddw, src, t1, t3);
330 emit_zext_32(ctx, src, true);
335 emit_insn(ctx, amandd, src, t1, t3);
337 emit_insn(ctx, amandw, src, t1, t3);
338 emit_zext_32(ctx, src, true);
343 emit_insn(ctx, amord, src, t1, t3);
345 emit_insn(ctx, amorw, src, t1, t3);
346 emit_zext_32(ctx, src, true);
351 emit_insn(ctx, amxord, src, t1, t3);
353 emit_insn(ctx, amxorw, src, t1, t3);
354 emit_zext_32(ctx, src, true);
360 emit_insn(ctx, amswapd, src, t1, t3);
362 emit_insn(ctx, amswapw, src, t1, t3);
363 emit_zext_32(ctx, src, true);
368 move_reg(ctx, t2, r0);
370 emit_insn(ctx, lld, r0, t1, 0);
371 emit_insn(ctx, bne, t2, r0, 4);
372 move_reg(ctx, t3, src);
373 emit_insn(ctx, scd, t3, t1, 0);
374 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
376 emit_insn(ctx, llw, r0, t1, 0);
377 emit_zext_32(ctx, t2, true);
378 emit_zext_32(ctx, r0, true);
379 emit_insn(ctx, bne, t2, r0, 4);
380 move_reg(ctx, t3, src);
381 emit_insn(ctx, scw, t3, t1, 0);
382 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
383 emit_zext_32(ctx, r0, true);
412 struct jit_ctx *ctx,
419 if (!ctx->image || !ctx->prog->aux->extable)
426 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries))
429 ex = &ctx->prog->aux->extable[ctx->num_exentries];
430 pc = (unsigned long)&ctx->image[ctx->idx - 1];
453 ctx->num_exentries++;
458 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
463 int i = insn - ctx->prog->insnsi;
481 move_reg(ctx, dst, src);
482 emit_zext_32(ctx, dst, is32);
485 move_reg(ctx, t1, src);
486 emit_insn(ctx, extwb, dst, t1);
487 emit_zext_32(ctx, dst, is32);
490 move_reg(ctx, t1, src);
491 emit_insn(ctx, extwh, dst, t1);
492 emit_zext_32(ctx, dst, is32);
495 emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO);
503 move_imm(ctx, dst, imm, is32);
509 emit_insn(ctx, addd, dst, dst, src);
510 emit_zext_32(ctx, dst, is32);
517 emit_insn(ctx, addid, dst, dst, imm);
519 move_imm(ctx, t1, imm, is32);
520 emit_insn(ctx, addd, dst, dst, t1);
522 emit_zext_32(ctx, dst, is32);
528 emit_insn(ctx, subd, dst, dst, src);
529 emit_zext_32(ctx, dst, is32);
536 emit_insn(ctx, addid, dst, dst, -imm);
538 move_imm(ctx, t1, imm, is32);
539 emit_insn(ctx, subd, dst, dst, t1);
541 emit_zext_32(ctx, dst, is32);
547 emit_insn(ctx, muld, dst, dst, src);
548 emit_zext_32(ctx, dst, is32);
554 move_imm(ctx, t1, imm, is32);
555 emit_insn(ctx, muld, dst, dst, t1);
556 emit_zext_32(ctx, dst, is32);
563 emit_zext_32(ctx, dst, is32);
564 move_reg(ctx, t1, src);
565 emit_zext_32(ctx, t1, is32);
566 emit_insn(ctx, divdu, dst, dst, t1);
567 emit_zext_32(ctx, dst, is32);
569 emit_sext_32(ctx, dst, is32);
570 move_reg(ctx, t1, src);
571 emit_sext_32(ctx, t1, is32);
572 emit_insn(ctx, divd, dst, dst, t1);
573 emit_sext_32(ctx, dst, is32);
581 move_imm(ctx, t1, imm, is32);
582 emit_zext_32(ctx, dst, is32);
583 emit_insn(ctx, divdu, dst, dst, t1);
584 emit_zext_32(ctx, dst, is32);
586 move_imm(ctx, t1, imm, false);
587 emit_sext_32(ctx, t1, is32);
588 emit_sext_32(ctx, dst, is32);
589 emit_insn(ctx, divd, dst, dst, t1);
590 emit_sext_32(ctx, dst, is32);
598 emit_zext_32(ctx, dst, is32);
599 move_reg(ctx, t1, src);
600 emit_zext_32(ctx, t1, is32);
601 emit_insn(ctx, moddu, dst, dst, t1);
602 emit_zext_32(ctx, dst, is32);
604 emit_sext_32(ctx, dst, is32);
605 move_reg(ctx, t1, src);
606 emit_sext_32(ctx, t1, is32);
607 emit_insn(ctx, modd, dst, dst, t1);
608 emit_sext_32(ctx, dst, is32);
616 move_imm(ctx, t1, imm, is32);
617 emit_zext_32(ctx, dst, is32);
618 emit_insn(ctx, moddu, dst, dst, t1);
619 emit_zext_32(ctx, dst, is32);
621 move_imm(ctx, t1, imm, false);
622 emit_sext_32(ctx, t1, is32);
623 emit_sext_32(ctx, dst, is32);
624 emit_insn(ctx, modd, dst, dst, t1);
625 emit_sext_32(ctx, dst, is32);
632 move_imm(ctx, t1, imm, is32);
633 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
634 emit_zext_32(ctx, dst, is32);
640 emit_insn(ctx, and, dst, dst, src);
641 emit_zext_32(ctx, dst, is32);
648 emit_insn(ctx, andi, dst, dst, imm);
650 move_imm(ctx, t1, imm, is32);
651 emit_insn(ctx, and, dst, dst, t1);
653 emit_zext_32(ctx, dst, is32);
659 emit_insn(ctx, or, dst, dst, src);
660 emit_zext_32(ctx, dst, is32);
667 emit_insn(ctx, ori, dst, dst, imm);
669 move_imm(ctx, t1, imm, is32);
670 emit_insn(ctx, or, dst, dst, t1);
672 emit_zext_32(ctx, dst, is32);
678 emit_insn(ctx, xor, dst, dst, src);
679 emit_zext_32(ctx, dst, is32);
686 emit_insn(ctx, xori, dst, dst, imm);
688 move_imm(ctx, t1, imm, is32);
689 emit_insn(ctx, xor, dst, dst, t1);
691 emit_zext_32(ctx, dst, is32);
696 emit_insn(ctx, sllw, dst, dst, src);
697 emit_zext_32(ctx, dst, is32);
701 emit_insn(ctx, slld, dst, dst, src);
706 emit_insn(ctx, slliw, dst, dst, imm);
707 emit_zext_32(ctx, dst, is32);
711 emit_insn(ctx, sllid, dst, dst, imm);
716 emit_insn(ctx, srlw, dst, dst, src);
717 emit_zext_32(ctx, dst, is32);
721 emit_insn(ctx, srld, dst, dst, src);
726 emit_insn(ctx, srliw, dst, dst, imm);
727 emit_zext_32(ctx, dst, is32);
731 emit_insn(ctx, srlid, dst, dst, imm);
736 emit_insn(ctx, sraw, dst, dst, src);
737 emit_zext_32(ctx, dst, is32);
741 emit_insn(ctx, srad, dst, dst, src);
746 emit_insn(ctx, sraiw, dst, dst, imm);
747 emit_zext_32(ctx, dst, is32);
751 emit_insn(ctx, sraid, dst, dst, imm);
759 emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
763 emit_zext_32(ctx, dst, is32);
775 emit_insn(ctx, revb2h, dst, dst);
777 emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
780 emit_insn(ctx, revb2w, dst, dst);
782 emit_zext_32(ctx, dst, true);
785 emit_insn(ctx, revbd, dst, dst);
811 jmp_offset = bpf2la_offset(i, off, ctx);
812 move_reg(ctx, t1, dst);
813 move_reg(ctx, t2, src);
815 emit_sext_32(ctx, t1, is32);
816 emit_sext_32(ctx, t2, is32);
818 emit_zext_32(ctx, t1, is32);
819 emit_zext_32(ctx, t2, is32);
821 if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
846 jmp_offset = bpf2la_offset(i, off, ctx);
848 move_imm(ctx, t1, imm, false);
854 move_reg(ctx, t2, dst);
856 emit_sext_32(ctx, tm, is32);
857 emit_sext_32(ctx, t2, is32);
859 emit_zext_32(ctx, tm, is32);
860 emit_zext_32(ctx, t2, is32);
862 if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
869 jmp_offset = bpf2la_offset(i, off, ctx);
870 emit_insn(ctx, and, t1, dst, src);
871 emit_zext_32(ctx, t1, is32);
872 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
879 jmp_offset = bpf2la_offset(i, off, ctx);
880 move_imm(ctx, t1, imm, is32);
881 emit_insn(ctx, and, t1, dst, t1);
882 emit_zext_32(ctx, t1, is32);
883 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
891 jmp_offset = bpf2la_offset(i, off, ctx);
893 jmp_offset = bpf2la_offset(i, imm, ctx);
894 if (emit_uncond_jmp(ctx, jmp_offset) < 0)
900 mark_call(ctx);
901 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
906 move_addr(ctx, t1, func_addr);
907 emit_insn(ctx, jirl, t1, LOONGARCH_GPR_RA, 0);
908 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
913 mark_tail_call(ctx);
914 if (emit_bpf_tail_call(ctx) < 0)
920 if (i == ctx->prog->len - 1)
923 jmp_offset = epilogue_offset(ctx);
924 if (emit_uncond_jmp(ctx, jmp_offset) < 0)
933 move_imm(ctx, dst, imm64, is32);
959 emit_insn(ctx, ldb, dst, src, off);
961 emit_insn(ctx, ldbu, dst, src, off);
963 move_imm(ctx, t1, off, is32);
965 emit_insn(ctx, ldxb, dst, src, t1);
967 emit_insn(ctx, ldxbu, dst, src, t1);
973 emit_insn(ctx, ldh, dst, src, off);
975 emit_insn(ctx, ldhu, dst, src, off);
977 move_imm(ctx, t1, off, is32);
979 emit_insn(ctx, ldxh, dst, src, t1);
981 emit_insn(ctx, ldxhu, dst, src, t1);
987 emit_insn(ctx, ldw, dst, src, off);
989 emit_insn(ctx, ldwu, dst, src, off);
991 move_imm(ctx, t1, off, is32);
993 emit_insn(ctx, ldxw, dst, src, t1);
995 emit_insn(ctx, ldxwu, dst, src, t1);
999 move_imm(ctx, t1, off, is32);
1000 emit_insn(ctx, ldxd, dst, src, t1);
1004 ret = add_exception_handler(insn, ctx, dst);
1016 move_imm(ctx, t1, imm, is32);
1018 emit_insn(ctx, stb, t1, dst, off);
1020 move_imm(ctx, t2, off, is32);
1021 emit_insn(ctx, stxb, t1, dst, t2);
1025 move_imm(ctx, t1, imm, is32);
1027 emit_insn(ctx, sth, t1, dst, off);
1029 move_imm(ctx, t2, off, is32);
1030 emit_insn(ctx, stxh, t1, dst, t2);
1034 move_imm(ctx, t1, imm, is32);
1036 emit_insn(ctx, stw, t1, dst, off);
1038 emit_insn(ctx, stptrw, t1, dst, off);
1040 move_imm(ctx, t2, off, is32);
1041 emit_insn(ctx, stxw, t1, dst, t2);
1045 move_imm(ctx, t1, imm, is32);
1047 emit_insn(ctx, std, t1, dst, off);
1049 emit_insn(ctx, stptrd, t1, dst, off);
1051 move_imm(ctx, t2, off, is32);
1052 emit_insn(ctx, stxd, t1, dst, t2);
1066 emit_insn(ctx, stb, src, dst, off);
1068 move_imm(ctx, t1, off, is32);
1069 emit_insn(ctx, stxb, src, dst, t1);
1074 emit_insn(ctx, sth, src, dst, off);
1076 move_imm(ctx, t1, off, is32);
1077 emit_insn(ctx, stxh, src, dst, t1);
1082 emit_insn(ctx, stw, src, dst, off);
1084 emit_insn(ctx, stptrw, src, dst, off);
1086 move_imm(ctx, t1, off, is32);
1087 emit_insn(ctx, stxw, src, dst, t1);
1092 emit_insn(ctx, std, src, dst, off);
1094 emit_insn(ctx, stptrd, src, dst, off);
1096 move_imm(ctx, t1, off, is32);
1097 emit_insn(ctx, stxd, src, dst, t1);
1105 emit_atomic(insn, ctx);
1124 static int build_body(struct jit_ctx *ctx, bool extra_pass)
1127 const struct bpf_prog *prog = ctx->prog;
1133 if (ctx->image == NULL)
1134 ctx->offset[i] = ctx->idx;
1136 ret = build_insn(insn, ctx, extra_pass);
1139 if (ctx->image == NULL)
1140 ctx->offset[i] = ctx->idx;
1147 if (ctx->image == NULL)
1148 ctx->offset[i] = ctx->idx;
1163 static int validate_code(struct jit_ctx *ctx)
1168 for (i = 0; i < ctx->idx; i++) {
1169 insn = ctx->image[i];
1175 if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
1186 struct jit_ctx ctx;
1221 if (jit_data->ctx.offset) {
1222 ctx = jit_data->ctx;
1226 prog_size = sizeof(u32) * ctx.idx;
1230 memset(&ctx, 0, sizeof(ctx));
1231 ctx.prog = prog;
1233 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
1234 if (ctx.offset == NULL) {
1239 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
1240 build_prologue(&ctx);
1241 if (build_body(&ctx, extra_pass)) {
1245 ctx.epilogue_offset = ctx.idx;
1246 build_epilogue(&ctx);
1255 prog_size = sizeof(u32) * ctx.idx;
1266 ctx.image = (union loongarch_instruction *)image_ptr;
1271 ctx.idx = 0;
1272 ctx.num_exentries = 0;
1274 build_prologue(&ctx);
1275 if (build_body(&ctx, extra_pass)) {
1280 build_epilogue(&ctx);
1283 if (validate_code(&ctx)) {
1291 bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
1294 flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx));
1297 if (extra_pass && ctx.idx != jit_data->ctx.idx) {
1299 ctx.idx, jit_data->ctx.idx);
1308 jit_data->ctx = ctx;
1314 prog->bpf_func = (void *)ctx.image;
1321 ctx.offset[i] *= LOONGARCH_INSN_SIZE;
1322 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
1325 kvfree(ctx.offset);