Lines Matching refs:ctx
271 static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx)
276 if (ctx->target != NULL)
277 ctx->target[ctx->idx] = inst;
279 ctx->idx++;
285 static inline void emit(u32 inst, struct jit_ctx *ctx)
287 _emit(ARM_COND_AL, inst, ctx);
404 #define _STACK_SIZE (ctx->prog->aux->stack_depth + SCRATCH_SIZE)
409 static u16 imm_offset(u32 k, struct jit_ctx *ctx)
415 if (ctx->target == NULL) {
416 ctx->imm_count++;
420 while ((i < ctx->imm_count) && ctx->imms[i]) {
421 if (ctx->imms[i] == k)
426 if (ctx->imms[i] == 0)
427 ctx->imms[i] = k;
430 offset = ctx->offsets[ctx->prog->len - 1] * 4;
431 offset += ctx->prologue_bytes;
432 offset += ctx->epilogue_bytes;
435 ctx->target[offset / 4] = k;
438 imm = offset - (8 + ctx->idx * 4);
445 ctx->flags |= FLAG_IMM_OVERFLOW;
455 const struct jit_ctx *ctx) {
458 if (ctx->target == NULL)
460 to = ctx->offsets[bpf_to];
461 from = ctx->offsets[bpf_from];
469 static inline void emit_mov_i_no8m(const u8 rd, u32 val, struct jit_ctx *ctx)
472 emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx);
474 emit(ARM_MOVW(rd, val & 0xffff), ctx);
476 emit(ARM_MOVT(rd, val >> 16), ctx);
480 static inline void emit_mov_i(const u8 rd, u32 val, struct jit_ctx *ctx)
485 emit(ARM_MOV_I(rd, imm12), ctx);
487 emit_mov_i_no8m(rd, val, ctx);
490 static void emit_bx_r(u8 tgt_reg, struct jit_ctx *ctx)
493 emit(ARM_BX(tgt_reg), ctx);
495 emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx);
498 static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx)
501 emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx);
502 emit_bx_r(tgt_reg, ctx);
504 emit(ARM_BLX_R(tgt_reg), ctx);
508 static inline int epilogue_offset(const struct jit_ctx *ctx)
512 if (ctx->target == NULL)
514 to = ctx->epilogue_offset;
515 from = ctx->idx;
520 static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op, u8 sign)
529 emit(sign ? ARM_SDIV(rd, rm, rn) : ARM_UDIV(rd, rm, rn), ctx);
531 emit(sign ? ARM_SDIV(ARM_IP, rm, rn) : ARM_UDIV(ARM_IP, rm, rn), ctx);
532 emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx);
547 emit(ARM_MOV_R(tmp[0], ARM_R1), ctx);
548 emit(ARM_MOV_R(ARM_R1, rn), ctx);
551 emit(ARM_MOV_R(tmp[1], ARM_R0), ctx);
552 emit(ARM_MOV_R(ARM_R0, rm), ctx);
556 emit(ARM_PUSH(CALLER_MASK & ~exclude_mask), ctx);
571 emit_mov_i(ARM_IP, dst, ctx);
572 emit_blx_r(ARM_IP, ctx);
575 emit(ARM_POP(CALLER_MASK & ~exclude_mask), ctx);
579 emit(ARM_MOV_R(rd, ARM_R0), ctx);
583 emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx);
585 emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx);
588 static inline void emit_udivmod64(const s8 *rd, const s8 *rm, const s8 *rn, struct jit_ctx *ctx,
594 emit(ARM_PUSH(CALLER_MASK), ctx);
607 emit(ARM_PUSH(BIT(ARM_R0) | BIT(ARM_R1)), ctx);
608 emit(ARM_MOV_R(ARM_R1, rm[0]), ctx);
609 emit(ARM_MOV_R(ARM_R0, rm[1]), ctx);
611 emit(ARM_POP(BIT(ARM_R2) | BIT(ARM_R3)), ctx);
619 emit(ARM_MOV_R(ARM_R3, rn[0]), ctx);
620 emit(ARM_MOV_R(ARM_R2, rn[1]), ctx);
639 emit_mov_i(ARM_IP, dst, ctx);
640 emit_blx_r(ARM_IP, ctx);
644 emit(ARM_MOV_R(rd[0], ARM_R1), ctx);
645 emit(ARM_MOV_R(rd[1], ARM_R0), ctx);
650 emit(ARM_POP(CALLER_MASK), ctx);
652 emit(ARM_POP(BIT(ARM_R0) | BIT(ARM_R1)), ctx);
653 emit(ARM_ADD_I(ARM_SP, ARM_SP, 8), ctx);
655 emit(ARM_ADD_I(ARM_SP, ARM_SP, 8), ctx);
656 emit(ARM_POP(BIT(ARM_R2) | BIT(ARM_R3)), ctx);
670 static s8 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx)
673 emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx);
680 struct jit_ctx *ctx)
684 ctx->cpu_architecture >= CPU_ARCH_ARMv5TE) {
686 EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx);
689 EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx);
691 EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx);
702 static void arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx)
705 emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx);
707 emit(ARM_MOV_R(reg, src), ctx);
711 struct jit_ctx *ctx)
715 ctx->cpu_architecture >= CPU_ARCH_ARMv5TE) {
717 EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx);
720 EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx);
722 EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx);
726 emit(ARM_MOV_R(reg[1], src[1]), ctx);
728 emit(ARM_MOV_R(reg[0], src[0]), ctx);
733 struct jit_ctx *ctx)
738 emit_mov_i(tmp[1], val, ctx);
739 arm_bpf_put_reg32(dst, tmp[1], ctx);
741 emit_mov_i(dst, val, ctx);
745 static void emit_a32_mov_i64(const s8 dst[], u64 val, struct jit_ctx *ctx)
750 emit_mov_i(rd[1], (u32)val, ctx);
751 emit_mov_i(rd[0], val >> 32, ctx);
753 arm_bpf_put_reg64(dst, rd, ctx);
758 const u32 val, struct jit_ctx *ctx) {
763 emit_a32_mov_i64(dst, val64, ctx);
768 struct jit_ctx *ctx) {
776 emit(ARM_ADDS_R(dst, dst, src), ctx);
778 emit(ARM_ADC_R(dst, dst, src), ctx);
780 emit(ARM_ADD_R(dst, dst, src), ctx);
785 struct jit_ctx *ctx) {
793 emit(ARM_SUBS_R(dst, dst, src), ctx);
795 emit(ARM_SBC_R(dst, dst, src), ctx);
797 emit(ARM_SUB_R(dst, dst, src), ctx);
801 const bool hi, const u8 op, struct jit_ctx *ctx){
805 emit_a32_add_r(dst, src, is64, hi, ctx);
809 emit_a32_sub_r(dst, src, is64, hi, ctx);
813 emit(ARM_ORR_R(dst, dst, src), ctx);
817 emit(ARM_AND_R(dst, dst, src), ctx);
821 emit(ARM_EOR_R(dst, dst, src), ctx);
825 emit(ARM_MUL(dst, dst, src), ctx);
829 emit(ARM_LSL_R(dst, dst, src), ctx);
833 emit(ARM_LSR_R(dst, dst, src), ctx);
837 emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx);
844 const s8 src[], struct jit_ctx *ctx,
850 rd = arm_bpf_get_reg64(dst, tmp, ctx);
854 rs = arm_bpf_get_reg64(src, tmp2, ctx);
857 emit_alu_r(rd[1], rs[1], true, false, op, ctx);
858 emit_alu_r(rd[0], rs[0], true, true, op, ctx);
862 rs = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
865 emit_alu_r(rd[1], rs, true, false, op, ctx);
866 if (!ctx->prog->aux->verifier_zext)
867 emit_a32_mov_i(rd[0], 0, ctx);
870 arm_bpf_put_reg64(dst, rd, ctx);
875 struct jit_ctx *ctx) {
879 rt = arm_bpf_get_reg32(src, tmp[0], ctx);
881 emit(ARM_LSL_I(rt, rt, 32 - off), ctx);
882 emit(ARM_ASR_I(rt, rt, 32 - off), ctx);
884 arm_bpf_put_reg32(dst, rt, ctx);
890 struct jit_ctx *ctx) {
892 emit_a32_mov_r(dst_lo, src_lo, 0, ctx);
893 if (!ctx->prog->aux->verifier_zext)
895 emit_a32_mov_i(dst_hi, 0, ctx);
897 ctx->cpu_architecture < CPU_ARCH_ARMv5TE) {
899 emit_a32_mov_r(dst_lo, src_lo, 0, ctx);
900 emit_a32_mov_r(dst_hi, src_hi, 0, ctx);
904 emit(ARM_LDRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx);
905 emit(ARM_STRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx);
907 emit(ARM_LDRD_I(dst[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx);
909 emit(ARM_STRD_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx);
911 emit(ARM_MOV_R(dst[0], src[0]), ctx);
912 emit(ARM_MOV_R(dst[1], src[1]), ctx);
918 struct jit_ctx *ctx) {
922 rt = arm_bpf_get_reg64(dst, tmp, ctx);
924 emit_a32_mov_r(dst_lo, src_lo, off, ctx);
926 if (!ctx->prog->aux->verifier_zext)
928 emit_a32_mov_i(dst_hi, 0, ctx);
930 emit(ARM_ASR_I(rt[0], rt[1], 31), ctx);
936 struct jit_ctx *ctx, const u8 op) {
940 rd = arm_bpf_get_reg32(dst, tmp[0], ctx);
945 emit(ARM_LSL_I(rd, rd, val), ctx);
948 emit(ARM_LSR_I(rd, rd, val), ctx);
951 emit(ARM_ASR_I(rd, rd, val), ctx);
954 emit(ARM_RSB_I(rd, rd, val), ctx);
958 arm_bpf_put_reg32(dst, rd, ctx);
963 struct jit_ctx *ctx){
968 rd = arm_bpf_get_reg64(dst, tmp, ctx);
971 emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx);
972 emit(ARM_RSC_I(rd[0], rd[0], 0), ctx);
974 arm_bpf_put_reg64(dst, rd, ctx);
979 struct jit_ctx *ctx) {
986 rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
987 rd = arm_bpf_get_reg64(dst, tmp, ctx);
990 emit(ARM_SUB_I(ARM_IP, rt, 32), ctx);
991 emit(ARM_RSB_I(tmp2[0], rt, 32), ctx);
992 emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx);
993 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx);
994 emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx);
995 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx);
997 arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
998 arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
1003 struct jit_ctx *ctx) {
1010 rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
1011 rd = arm_bpf_get_reg64(dst, tmp, ctx);
1014 emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
1015 emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
1016 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);
1017 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);
1019 ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASR, tmp2[0]), ctx);
1020 emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx);
1022 arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
1023 arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
1028 struct jit_ctx *ctx) {
1035 rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
1036 rd = arm_bpf_get_reg64(dst, tmp, ctx);
1039 emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);
1040 emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);
1041 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);
1042 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);
1043 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx);
1044 emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx);
1046 arm_bpf_put_reg32(dst_lo, ARM_LR, ctx);
1047 arm_bpf_put_reg32(dst_hi, ARM_IP, ctx);
1052 const u32 val, struct jit_ctx *ctx){
1058 rd = arm_bpf_get_reg64(dst, tmp, ctx);
1062 emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx);
1063 emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx);
1064 emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx);
1067 emit(ARM_MOV_R(rd[0], rd[1]), ctx);
1069 emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx);
1070 emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx);
1073 arm_bpf_put_reg64(dst, rd, ctx);
1078 const u32 val, struct jit_ctx *ctx) {
1084 rd = arm_bpf_get_reg64(dst, tmp, ctx);
1092 emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx);
1093 emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx);
1094 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx);
1096 emit(ARM_MOV_R(rd[1], rd[0]), ctx);
1097 emit(ARM_MOV_I(rd[0], 0), ctx);
1099 emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx);
1100 emit(ARM_MOV_I(rd[0], 0), ctx);
1103 arm_bpf_put_reg64(dst, rd, ctx);
1108 const u32 val, struct jit_ctx *ctx){
1114 rd = arm_bpf_get_reg64(dst, tmp, ctx);
1122 emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx);
1123 emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx);
1124 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx);
1126 emit(ARM_MOV_R(rd[1], rd[0]), ctx);
1127 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx);
1129 emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx);
1130 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx);
1133 arm_bpf_put_reg64(dst, rd, ctx);
1137 struct jit_ctx *ctx) {
1143 rd = arm_bpf_get_reg64(dst, tmp, ctx);
1144 rt = arm_bpf_get_reg64(src, tmp2, ctx);
1147 emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx);
1148 emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx);
1149 emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx);
1151 emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx);
1152 emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx);
1154 arm_bpf_put_reg32(dst_lo, ARM_IP, ctx);
1155 arm_bpf_put_reg32(dst_hi, rd[0], ctx);
1198 s16 off, struct jit_ctx *ctx, const u8 sz){
1202 rd = arm_bpf_get_reg32(dst, tmp[1], ctx);
1205 emit_a32_mov_i(tmp[0], off, ctx);
1206 emit(ARM_ADD_R(tmp[0], tmp[0], rd), ctx);
1213 emit(ARM_STRB_I(src_lo, rd, off), ctx);
1217 emit(ARM_STRH_I(src_lo, rd, off), ctx);
1221 emit(ARM_STR_I(src_lo, rd, off), ctx);
1225 emit(ARM_STR_I(src_lo, rd, off), ctx);
1226 emit(ARM_STR_I(src_hi, rd, off + 4), ctx);
1233 s16 off, struct jit_ctx *ctx, const u8 sz){
1239 emit_a32_mov_i(tmp[0], off, ctx);
1240 emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx);
1244 emit(ARM_MOV_R(tmp[0], rm), ctx);
1250 emit(ARM_LDRB_I(rd[1], rm, off), ctx);
1251 if (!ctx->prog->aux->verifier_zext)
1252 emit_a32_mov_i(rd[0], 0, ctx);
1256 emit(ARM_LDRH_I(rd[1], rm, off), ctx);
1257 if (!ctx->prog->aux->verifier_zext)
1258 emit_a32_mov_i(rd[0], 0, ctx);
1262 emit(ARM_LDR_I(rd[1], rm, off), ctx);
1263 if (!ctx->prog->aux->verifier_zext)
1264 emit_a32_mov_i(rd[0], 0, ctx);
1268 emit(ARM_LDR_I(rd[1], rm, off), ctx);
1269 emit(ARM_LDR_I(rd[0], rm, off + 4), ctx);
1272 arm_bpf_put_reg64(dst, rd, ctx);
1277 s16 off, struct jit_ctx *ctx, const u8 sz){
1290 emit(ARM_ADD_I(tmp[0], src, add_off), ctx);
1293 emit_a32_mov_i(tmp[0], off, ctx);
1294 emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx);
1303 emit(ARM_LDRSB_I(rd[1], rm, off), ctx);
1307 emit(ARM_LDRSH_I(rd[1], rm, off), ctx);
1311 emit(ARM_LDR_I(rd[1], rm, off), ctx);
1315 emit(ARM_ASR_I(rd[0], rd[1], 31), ctx);
1316 arm_bpf_put_reg64(dst, rd, ctx);
1321 const u8 rn, struct jit_ctx *ctx, u8 op,
1326 emit(ARM_AND_R(ARM_IP, rt, rn), ctx);
1327 emit(ARM_AND_R(ARM_LR, rd, rm), ctx);
1328 emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx);
1330 emit(ARM_ANDS_R(ARM_IP, rt, rn), ctx);
1340 emit(ARM_CMP_R(rd, rm), ctx);
1342 _emit(ARM_COND_EQ, ARM_CMP_R(rt, rn), ctx);
1344 emit(ARM_CMP_R(rt, rn), ctx);
1349 emit(ARM_CMP_R(rn, rt), ctx);
1351 emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx);
1355 emit(ARM_CMP_R(rt, rn), ctx);
1357 emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx);
1363 static int emit_bpf_tail_call(struct jit_ctx *ctx)
1373 const int idx0 = ctx->idx;
1374 #define cur_offset (ctx->idx - idx0)
1386 r_array = arm_bpf_get_reg32(r2[1], tmp2[0], ctx);
1388 r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx);
1390 emit(ARM_LDR_I(tmp[1], r_array, off), ctx);
1392 emit(ARM_CMP_R(r_index, tmp[1]), ctx);
1393 _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx);
1404 tc = arm_bpf_get_reg64(tcc, tmp, ctx);
1405 emit(ARM_CMP_I(tc[0], hi), ctx);
1406 _emit(ARM_COND_EQ, ARM_CMP_I(tc[1], lo), ctx);
1407 _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx);
1408 emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx);
1409 emit(ARM_ADC_I(tc[0], tc[0], 0), ctx);
1410 arm_bpf_put_reg64(tcc, tmp, ctx);
1418 emit(ARM_ADD_I(tmp[1], r_array, off), ctx);
1419 emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx);
1420 emit(ARM_CMP_I(tmp[1], 0), ctx);
1421 _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
1427 emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx);
1428 emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx);
1429 emit_bx_r(tmp[1], ctx);
1445 static inline void emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx)
1450 emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);
1451 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx);
1452 emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);
1453 emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx);
1455 emit(ARM_REV16(rd, rn), ctx);
1460 static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx)
1465 emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);
1466 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx);
1467 emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx);
1469 emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx);
1470 emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx);
1471 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx);
1472 emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);
1473 emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx);
1474 emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx);
1475 emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx);
1478 emit(ARM_REV(rd, rn), ctx);
1483 static inline void emit_push_r64(const s8 src[], struct jit_ctx *ctx)
1489 rt = arm_bpf_get_reg64(src, tmp2, ctx);
1492 emit(ARM_PUSH(reg_set), ctx);
1495 static void build_prologue(struct jit_ctx *ctx)
1505 emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx);
1506 emit(ARM_PUSH(reg_set), ctx);
1507 emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx);
1509 emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx);
1510 emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx);
1514 emit(ARM_MOV_I(bpf_r1[0], 0), ctx);
1515 emit(ARM_SUB_I(bpf_r1[1], ARM_SP, SCRATCH_SIZE), ctx);
1517 ctx->stack_size = imm8m(STACK_SIZE);
1520 emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx);
1523 emit_a32_mov_r64(true, bpf_fp, bpf_r1, ctx);
1526 emit(ARM_MOV_I(bpf_r1[1], 0), ctx);
1527 emit_a32_mov_r64(true, tcc, bpf_r1, ctx);
1530 emit(ARM_MOV_R(bpf_r1[1], arm_r0), ctx);
1536 static void build_epilogue(struct jit_ctx *ctx)
1542 emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx);
1543 emit(ARM_LDM(ARM_SP, reg_set), ctx);
1546 emit(ARM_MOV_R(ARM_SP, ARM_FP), ctx);
1547 emit(ARM_POP(CALLEE_POP_MASK), ctx);
1559 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
1568 const int i = insn - ctx->prog->insnsi;
1596 emit_a32_mov_i(dst_hi, 0, ctx);
1600 emit_a32_movsx_r64(is64, insn->off, dst, src, ctx);
1602 emit_a32_mov_r64(is64, dst, src, ctx);
1606 emit_a32_mov_se_i64(is64, dst, imm, ctx);
1645 emit_a32_alu_r64(is64, dst, src, ctx, BPF_OP(code));
1654 emit_a32_mov_se_i64(is64, tmp2, imm, ctx);
1655 emit_a32_alu_r64(is64, dst, tmp2, ctx, BPF_OP(code));
1665 rd_lo = arm_bpf_get_reg32(dst_lo, tmp2[1], ctx);
1668 rt = arm_bpf_get_reg32(src_lo, tmp2[0], ctx);
1672 emit_a32_mov_i(rt, imm, ctx);
1678 emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code), off);
1679 arm_bpf_put_reg32(dst_lo, rd_lo, ctx);
1680 if (!ctx->prog->aux->verifier_zext)
1681 emit_a32_mov_i(dst_hi, 0, ctx);
1687 rd = arm_bpf_get_reg64(dst, tmp2, ctx);
1690 rs = arm_bpf_get_reg64(src, tmp, ctx);
1694 emit_a32_mov_se_i64(is64, rs, imm, ctx);
1697 emit_udivmod64(rd, rd, rs, ctx, BPF_OP(code), off);
1698 arm_bpf_put_reg64(dst, rd, ctx);
1709 emit_a32_alu_i(dst_lo, imm, ctx, BPF_OP(code));
1710 if (!ctx->prog->aux->verifier_zext)
1711 emit_a32_mov_i(dst_hi, 0, ctx);
1717 emit_a32_lsh_i64(dst, imm, ctx);
1723 emit_a32_rsh_i64(dst, imm, ctx);
1727 emit_a32_lsh_r64(dst, src, ctx);
1731 emit_a32_rsh_r64(dst, src, ctx);
1735 emit_a32_arsh_r64(dst, src, ctx);
1741 emit_a32_arsh_i64(dst, imm, ctx);
1745 emit_a32_alu_i(dst_lo, 0, ctx, BPF_OP(code));
1746 if (!ctx->prog->aux->verifier_zext)
1747 emit_a32_mov_i(dst_hi, 0, ctx);
1751 emit_a32_neg64(dst, ctx);
1758 emit_a32_mul_r64(dst, src, ctx);
1767 emit_a32_mov_se_i64(is64, tmp2, imm, ctx);
1768 emit_a32_mul_r64(dst, tmp2, ctx);
1778 rd = arm_bpf_get_reg64(dst, tmp, ctx);
1783 emit_rev16(rd[1], rd[1], ctx);
1786 emit_rev32(rd[1], rd[1], ctx);
1789 emit_rev32(ARM_LR, rd[1], ctx);
1790 emit_rev32(rd[1], rd[0], ctx);
1791 emit(ARM_MOV_R(rd[0], ARM_LR), ctx);
1800 emit_a32_mov_i(tmp2[1], 0xffff, ctx);
1801 emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx);
1803 emit(ARM_UXTH(rd[1], rd[1]), ctx);
1805 if (!ctx->prog->aux->verifier_zext)
1806 emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
1810 if (!ctx->prog->aux->verifier_zext)
1811 emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);
1818 arm_bpf_put_reg64(dst, rd, ctx);
1825 emit_a32_mov_i64(dst, val, ctx);
1838 rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
1840 emit_ldsx_r(dst, rn, off, ctx, BPF_SIZE(code));
1842 emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code));
1855 emit_a32_mov_se_i64(true, tmp2, imm, ctx);
1860 emit_a32_mov_i(tmp2[1], imm, ctx);
1863 emit_str_r(dst_lo, tmp2, off, ctx, BPF_SIZE(code));
1874 rs = arm_bpf_get_reg64(src, tmp2, ctx);
1875 emit_str_r(dst_lo, rs, off, ctx, BPF_SIZE(code));
1911 rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx);
1912 rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx);
1952 emit_a32_mov_se_i64(true, tmp2, imm, ctx);
1955 rd = arm_bpf_get_reg64(dst, tmp, ctx);
1958 emit_ar_r(rd[0], rd[1], rm, rn, ctx, BPF_OP(code),
1962 jmp_offset = bpf2a32_offset(i+off, i, ctx);
1966 _emit(ARM_COND_NE, ARM_B(jmp_offset), ctx);
1969 _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
1972 _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx);
1975 _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx);
1978 _emit(ARM_COND_LT, ARM_B(jmp_offset), ctx);
1981 _emit(ARM_COND_GE, ARM_B(jmp_offset), ctx);
1984 _emit(ARM_COND_LS, ARM_B(jmp_offset), ctx);
1987 _emit(ARM_COND_CC, ARM_B(jmp_offset), ctx);
1990 _emit(ARM_COND_LT, ARM_B(jmp_offset), ctx);
1993 _emit(ARM_COND_GE, ARM_B(jmp_offset), ctx);
2002 jmp_offset = bpf2a32_offset(i + imm, i, ctx);
2004 jmp_offset = bpf2a32_offset(i + off, i, ctx);
2009 emit(ARM_B(jmp_offset), ctx);
2014 if (emit_bpf_tail_call(ctx))
2028 emit_a32_mov_r64(true, r0, r1, ctx);
2029 emit_a32_mov_r64(true, r1, r2, ctx);
2030 emit_push_r64(r5, ctx);
2031 emit_push_r64(r4, ctx);
2032 emit_push_r64(r3, ctx);
2034 emit_a32_mov_i(tmp[1], func, ctx);
2035 emit_blx_r(tmp[1], ctx);
2037 emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean
2045 if (i == ctx->prog->len - 1)
2047 jmp_offset = epilogue_offset(ctx);
2049 emit(ARM_B(jmp_offset), ctx);
2059 if (ctx->flags & FLAG_IMM_OVERFLOW)
2069 static int build_body(struct jit_ctx *ctx)
2071 const struct bpf_prog *prog = ctx->prog;
2078 ret = build_insn(insn, ctx);
2083 if (ctx->target == NULL)
2084 ctx->offsets[i] = ctx->idx;
2088 if (ctx->target == NULL)
2089 ctx->offsets[i] = ctx->idx;
2098 static int validate_code(struct jit_ctx *ctx)
2102 for (i = 0; i < ctx->idx; i++) {
2103 if (ctx->target[i] == __opcode_to_mem_arm(ARM_INST_UDF))
2120 struct jit_ctx ctx;
2144 memset(&ctx, 0, sizeof(ctx));
2145 ctx.prog = prog;
2146 ctx.cpu_architecture = cpu_architecture();
2151 ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
2152 if (ctx.offsets == NULL) {
2158 * to compute ctx->offsets and other context variables
2167 if (build_body(&ctx)) {
2172 tmp_idx = ctx.idx;
2173 build_prologue(&ctx);
2174 ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4;
2176 ctx.epilogue_offset = ctx.idx;
2179 tmp_idx = ctx.idx;
2180 build_epilogue(&ctx);
2181 ctx.epilogue_bytes = (ctx.idx - tmp_idx) * 4;
2183 ctx.idx += ctx.imm_count;
2184 if (ctx.imm_count) {
2185 ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL);
2186 if (ctx.imms == NULL) {
2193 build_epilogue(&ctx);
2203 image_size = sizeof(u32) * ctx.idx;
2217 ctx.target = (u32 *) image_ptr;
2218 ctx.idx = 0;
2220 build_prologue(&ctx);
2225 if (build_body(&ctx) < 0) {
2231 build_epilogue(&ctx);
2234 if (validate_code(&ctx)) {
2240 flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx));
2244 bpf_jit_dump(prog->len, image_size, 2, ctx.target);
2247 prog->bpf_func = (void *)ctx.target;
2253 if (ctx.imm_count)
2254 kfree(ctx.imms);
2257 kfree(ctx.offsets);