H A D | bpf_jit_32.c | 271 static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx) argument 276 if (ctx->target != NULL) 277 ctx->target[ctx->idx] = inst; 279 ctx->idx++; 285 static inline void emit(u32 inst, struct jit_ctx *ctx) argument 287 _emit(ARM_COND_AL, inst, ctx); 404 #define _STACK_SIZE (ctx->prog->aux->stack_depth + SCRATCH_SIZE) 409 static u16 imm_offset(u32 k, struct jit_ctx *ctx) argument 415 if (ctx 454 bpf2a32_offset(int bpf_to, int bpf_from, const struct jit_ctx *ctx) argument 469 emit_mov_i_no8m(const u8 rd, u32 val, struct jit_ctx *ctx) argument 472 emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx); local 480 emit_mov_i(const u8 rd, u32 val, struct jit_ctx *ctx) argument 485 emit(ARM_MOV_I(rd, imm12), ctx); local 490 emit_bx_r(u8 tgt_reg, struct jit_ctx *ctx) argument 493 emit(ARM_BX(tgt_reg), ctx); local 495 emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx); local 498 emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx) argument 501 emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx); local 508 epilogue_offset(const struct jit_ctx *ctx) argument 520 emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op, u8 sign) argument 532 emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx); local 547 emit(ARM_MOV_R(tmp[0], ARM_R1), ctx); local 548 emit(ARM_MOV_R(ARM_R1, rn), ctx); local 551 emit(ARM_MOV_R(tmp[1], ARM_R0), ctx); local 552 emit(ARM_MOV_R(ARM_R0, rm), ctx); local 556 emit(ARM_PUSH(CALLER_MASK & ~exclude_mask), ctx); local 575 emit(ARM_POP(CALLER_MASK & ~exclude_mask), ctx); local 579 emit(ARM_MOV_R(rd, ARM_R0), ctx); local 583 emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx); local 585 emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx); local 588 emit_udivmod64(const s8 *rd, const s8 *rm, const s8 *rn, struct jit_ctx *ctx, u8 op, u8 sign) argument 594 emit(ARM_PUSH(CALLER_MASK), ctx); local 608 emit(ARM_MOV_R(ARM_R1, rm[0]), ctx); local 609 emit(ARM_MOV_R(ARM_R0, rm[1]), ctx); local 619 emit(ARM_MOV_R(ARM_R3, rn[0]), ctx); local 620 emit(ARM_MOV_R(ARM_R2, rn[1]), ctx); local 644 emit(ARM_MOV_R(rd[0], ARM_R1), ctx); local 645 emit(ARM_MOV_R(rd[1], ARM_R0), ctx); local 650 emit(ARM_POP(CALLER_MASK), ctx); local 670 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx) argument 673 emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); local 679 arm_bpf_get_reg64(const s8 *reg, const s8 *tmp, struct jit_ctx *ctx) argument 686 EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); local 689 EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); local 691 EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx); local 702 arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx) argument 705 emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); local 707 emit(ARM_MOV_R(reg, src), ctx); local 710 arm_bpf_put_reg64(const s8 *reg, const s8 *src, struct jit_ctx *ctx) argument 717 EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); local 720 EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); local 722 EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx); local 726 emit(ARM_MOV_R(reg[1], src[1]), ctx); local 728 emit(ARM_MOV_R(reg[0], src[0]), ctx); local 732 emit_a32_mov_i(const s8 dst, const u32 val, struct jit_ctx *ctx) argument 745 emit_a32_mov_i64(const s8 dst[], u64 val, struct jit_ctx *ctx) argument 757 emit_a32_mov_se_i64(const bool is64, const s8 dst[], const u32 val, struct jit_ctx *ctx) argument 766 emit_a32_add_r(const u8 dst, const u8 src, const bool is64, const bool hi, struct jit_ctx *ctx) argument 776 emit(ARM_ADDS_R(dst, dst, src), ctx); local 778 emit(ARM_ADC_R(dst, dst, src), ctx); local 780 emit(ARM_ADD_R(dst, dst, src), ctx); local 783 emit_a32_sub_r(const u8 dst, const u8 src, const bool is64, const bool hi, struct jit_ctx *ctx) argument 793 emit(ARM_SUBS_R(dst, dst, src), ctx); local 795 emit(ARM_SBC_R(dst, dst, src), ctx); local 797 emit(ARM_SUB_R(dst, dst, src), ctx); local 800 emit_alu_r(const u8 dst, const u8 src, const bool is64, const bool hi, const u8 op, struct jit_ctx *ctx) argument 813 emit(ARM_ORR_R(dst, dst, src), ctx); local 817 emit(ARM_AND_R(dst, dst, src), ctx); local 821 emit(ARM_EOR_R(dst, dst, src), ctx); local 825 emit(ARM_MUL(dst, dst, src), ctx); local 829 emit(ARM_LSL_R(dst, dst, src), ctx); local 833 emit(ARM_LSR_R(dst, dst, src), ctx); local 837 emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx); local 843 emit_a32_alu_r64(const bool is64, const s8 dst[], const s8 src[], struct jit_ctx *ctx, const u8 op) argument 874 emit_a32_mov_r(const s8 dst, const s8 src, const u8 off, struct jit_ctx *ctx) argument 888 emit_a32_mov_r64(const bool is64, const s8 dst[], const s8 src[], struct jit_ctx *ctx) argument 904 emit(ARM_LDRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); local 905 emit(ARM_STRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); local 907 emit(ARM_LDRD_I(dst[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); local 909 emit(ARM_STRD_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); local 911 emit(ARM_MOV_R(dst[0], src[0]), ctx); local 912 emit(ARM_MOV_R(dst[1], src[1]), ctx); local 917 emit_a32_movsx_r64(const bool is64, const u8 off, const s8 dst[], const s8 src[], struct jit_ctx *ctx) argument 930 emit(ARM_ASR_I(rt[0], rt[1], 31), ctx); local 935 emit_a32_alu_i(const s8 dst, const u32 val, struct jit_ctx *ctx, const u8 op) argument 945 emit(ARM_LSL_I(rd, rd, val), ctx); local 948 emit(ARM_LSR_I(rd, rd, val), ctx); local 951 emit(ARM_ASR_I(rd, rd, val), ctx); local 954 emit(ARM_RSB_I(rd, rd, val), ctx); local 962 emit_a32_neg64(const s8 dst[], struct jit_ctx *ctx) argument 971 emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx); local 972 emit(ARM_RSC_I(rd[0], rd[0], 0), ctx); local 978 emit_a32_lsh_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) argument 991 emit(ARM_RSB_I(tmp2[0], rt, 32), ctx); local 992 emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx); local 993 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx); local 994 emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx); local 995 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx); local 1002 emit_a32_arsh_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) argument 1015 emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); local 1016 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); local 1017 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); local 1020 emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx); local 1027 emit_a32_rsh_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) argument 1040 emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); local 1041 emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); local 1042 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); local 1043 emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx); local 1044 emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx); local 1051 emit_a32_lsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx) argument 1062 emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx); local 1063 emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx); local 1064 emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx); local 1067 emit(ARM_MOV_R(rd[0], rd[1]), ctx); local 1069 emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx); local 1070 emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx); local 1077 emit_a32_rsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx) argument 1092 emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); local 1093 emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); local 1094 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx); local 1096 emit(ARM_MOV_R(rd[1], rd[0]), ctx); local 1097 emit(ARM_MOV_I(rd[0], 0), ctx); local 1099 emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx); local 1100 emit(ARM_MOV_I(rd[0], 0), ctx); local 1107 emit_a32_arsh_i64(const s8 dst[], const u32 val, struct jit_ctx *ctx) argument 1122 emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); local 1123 emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); local 1124 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx); local 1126 emit(ARM_MOV_R(rd[1], rd[0]), ctx); local 1127 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); local 1129 emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx); local 1130 emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); local 1136 emit_a32_mul_r64(const s8 dst[], const s8 src[], struct jit_ctx *ctx) argument 1147 emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx); local 1148 emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx); local 1149 emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx); local 1151 emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx); local 1152 emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx); local 1197 emit_str_r(const s8 dst, const s8 src[], s16 off, struct jit_ctx *ctx, const u8 sz) argument 1206 emit(ARM_ADD_R(tmp[0], tmp[0], rd), ctx); local 1213 emit(ARM_STRB_I(src_lo, rd, off), ctx); local 1217 emit(ARM_STRH_I(src_lo, rd, off), ctx); local 1221 emit(ARM_STR_I(src_lo, rd, off), ctx); local 1225 emit(ARM_STR_I(src_lo, rd, off), ctx); local 1232 emit_ldx_r(const s8 dst[], const s8 src, s16 off, struct jit_ctx *ctx, const u8 sz) argument 1240 emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx); local 1244 emit(ARM_MOV_R(tmp[0], rm), ctx); local 1250 emit(ARM_LDRB_I(rd[1], rm, off), ctx); local 1256 emit(ARM_LDRH_I(rd[1], rm, off), ctx); local 1262 emit(ARM_LDR_I(rd[1], rm, off), ctx); local 1268 emit(ARM_LDR_I(rd[1], rm, off), ctx); local 1269 emit(ARM_LDR_I(rd[0], rm, off + 4), ctx); local 1276 emit_ldsx_r(const s8 dst[], const s8 src, s16 off, struct jit_ctx *ctx, const u8 sz) argument 1290 emit(ARM_ADD_I(tmp[0], src, add_off), ctx); local 1294 emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx); local 1303 emit(ARM_LDRSB_I(rd[1], rm, off), ctx); local 1307 emit(ARM_LDRSH_I(rd[1], rm, off), ctx); local 1311 emit(ARM_LDR_I(rd[1], rm, off), ctx); local 1315 emit(ARM_ASR_I(rd[0], rd[1], 31), ctx); local 1320 emit_ar_r(const u8 rd, const u8 rt, const u8 rm, const u8 rn, struct jit_ctx *ctx, u8 op, bool is_jmp64) argument 1326 emit(ARM_AND_R(ARM_IP, rt, rn), ctx); local 1327 emit(ARM_AND_R(ARM_LR, rd, rm), ctx); local 1328 emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx); local 1330 emit(ARM_ANDS_R(ARM_IP, rt, rn), ctx); local 1340 emit(ARM_CMP_R(rd, rm), ctx); local 1344 emit(ARM_CMP_R(rt, rn), ctx); local 1349 emit(ARM_CMP_R(rn, rt), ctx); local 1351 emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx); local 1355 emit(ARM_CMP_R(rt, rn), ctx); local 1357 emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx); local 1363 emit_bpf_tail_call(struct jit_ctx *ctx) argument 1390 emit(ARM_LDR_I(tmp[1], r_array, off), ctx); local 1392 emit(ARM_CMP_R(r_index, tmp[1]), ctx); local 1405 emit(ARM_CMP_I(tc[0], hi), ctx); local 1408 emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx); local 1409 emit(ARM_ADC_I(tc[0], tc[0], 0), ctx); local 1418 emit(ARM_ADD_I(tmp[1], r_array, off), ctx); local 1419 emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx); local 1420 emit(ARM_CMP_I(tmp[1], 0), ctx); local 1427 emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx); local 1428 emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx); local 1445 emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx) argument 1450 emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); local 1451 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx); local 1452 emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx); local 1453 emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx); local 1460 emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx) argument 1465 emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); local 1466 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx); local 1467 emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx); local 1469 emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx); local 1470 emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx); local 1471 emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx); local 1472 emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx); local 1473 emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx); local 1474 emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx); local 1475 emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx); local 1483 emit_push_r64(const s8 src[], struct jit_ctx *ctx) argument 1492 emit(ARM_PUSH(reg_set), ctx); local 1495 build_prologue(struct jit_ctx *ctx) argument 1505 emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx); local 1506 emit(ARM_PUSH(reg_set), ctx); local 1509 emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx); local 1510 emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx); local 1514 emit(ARM_MOV_I(bpf_r1[0], 0), ctx); local 1515 emit(ARM_SUB_I(bpf_r1[1], ARM_SP, SCRATCH_SIZE), ctx); local 1526 emit(ARM_MOV_I(bpf_r1[1], 0), ctx); local 1530 emit(ARM_MOV_R(bpf_r1[1], arm_r0), ctx); local 1536 build_epilogue(struct jit_ctx *ctx) argument 1542 emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx); local 1543 emit(ARM_LDM(ARM_SP, reg_set), ctx); local 1559 build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) argument 1791 emit(ARM_MOV_R(rd[0], ARM_LR), ctx); local 1801 emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx); local 1806 emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); local 1811 emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); local 2009 emit(ARM_B(jmp_offset), ctx); local 2037 emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean local 2049 emit(ARM_B(jmp_offset), ctx); local 2069 build_body(struct jit_ctx *ctx) argument 2098 validate_code(struct jit_ctx *ctx) argument 2120 struct jit_ctx ctx; local [all...] |