arm/net/bpf_jit_32.c

70  * When popping registers off the stack at the end of a BPF function, we
371 #define ARM_LDR_I(rt, rn, off)	arm_bpf_ldst_imm12(ARM_INST_LDR_I, rt, rn, off)
372 #define ARM_LDRB_I(rt, rn, off)	arm_bpf_ldst_imm12(ARM_INST_LDRB_I, rt, rn, off)
373 #define ARM_LDRD_I(rt, rn, off)	arm_bpf_ldst_imm8(ARM_INST_LDRD_I, rt, rn, off)
374 #define ARM_LDRH_I(rt, rn, off)	arm_bpf_ldst_imm8(ARM_INST_LDRH_I, rt, rn, off)
376 #define ARM_LDRSH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRSH_I, rt, rn, off)
377 #define ARM_LDRSB_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRSB_I, rt, rn, off)
379 #define ARM_STR_I(rt, rn, off)	arm_bpf_ldst_imm12(ARM_INST_STR_I, rt, rn, off)
380 #define ARM_STRB_I(rt, rn, off)	arm_bpf_ldst_imm12(ARM_INST_STRB_I, rt, rn, off)
381 #define ARM_STRD_I(rt, rn, off)	arm_bpf_ldst_imm8(ARM_INST_STRD_I, rt, rn, off)
382 #define ARM_STRH_I(rt, rn, off)	arm_bpf_ldst_imm8(ARM_INST_STRH_I, rt, rn, off)
912 static inline void emit_a32_movsx_r64(const bool is64, const u8 off, const s8 dst[], const s8 src[],
925 	/* Sign extend rs if needed. If off == 32, lower 32-bits of src are moved to dst and sign
928 	if (off != 32) {
930 		emit(ARM_LSL_I(rd, rs, 32 - off), ctx);
931 		emit(ARM_ASR_I(rd, rd, 32 - off), ctx);
941 	 * 2. off == 32
943 	 * In this case src_lo was loaded into rd(tmp[1]) but rd was not sign extended as off==32.
947 	if (dst_lo != src_lo || off != 32)
1188 static bool is_ldst_imm(s16 off, const u8 size)
1201 		/* Need to make sure off+4 does not overflow. */
1205 	return -off_max <= off && off <= off_max;
1208 static bool is_ldst_imm8(s16 off, const u8 size)
1223 	return -off_max <= off && off <= off_max;
1226 /* *(size *)(dst + off) = src */
1228 			      s16 off, struct jit_ctx *ctx, const u8 sz){
1234 	if (!is_ldst_imm(off, sz)) {
1235 		emit_a32_mov_i(tmp[0], off, ctx);
1238 		off = 0;
1243 		emit(ARM_STRB_I(src_lo, rd, off), ctx);
1247 		emit(ARM_STRH_I(src_lo, rd, off), ctx);
1251 		emit(ARM_STR_I(src_lo, rd, off), ctx);
1255 		emit(ARM_STR_I(src_lo, rd, off), ctx);
1256 		emit(ARM_STR_I(src_hi, rd, off + 4), ctx);
1261 /* dst = *(size*)(src + off) */
1263 			      s16 off, struct jit_ctx *ctx, const u8 sz){
1268 	if (!is_ldst_imm(off, sz)) {
1269 		emit_a32_mov_i(tmp[0], off, ctx);
1272 		off = 0;
1280 		emit(ARM_LDRB_I(rd[1], rm, off), ctx);
1286 		emit(ARM_LDRH_I(rd[1], rm, off), ctx);
1292 		emit(ARM_LDR_I(rd[1], rm, off), ctx);
1298 		emit(ARM_LDR_I(rd[1], rm, off), ctx);
1299 		emit(ARM_LDR_I(rd[0], rm, off + 4), ctx);
1305 /* dst = *(signed size*)(src + off) */
1307 			       s16 off, struct jit_ctx *ctx, const u8 sz){
1313 	if (!is_ldst_imm8(off, sz)) {
1318 		add_off = imm8m(off);
1323 			emit_a32_mov_i(tmp[0], off, ctx);
1327 		off = 0;
1333 		emit(ARM_LDRSB_I(rd[1], rm, off), ctx);
1337 		emit(ARM_LDRSH_I(rd[1], rm, off), ctx);
1341 		emit(ARM_LDR_I(rd[1], rm, off), ctx);
1408 	int off;
1415 	off = offsetof(struct bpf_array, map.max_entries);
1420 	emit(ARM_LDR_I(tmp[1], r_array, off), ctx);
1447 	off = imm8m(offsetof(struct bpf_array, ptrs));
1448 	emit(ARM_ADD_I(tmp[1], r_array, off), ctx);
1456 	off = offsetof(struct bpf_prog, bpf_func);
1457 	emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx);
1596 	const s16 off = insn->off;
1629 			if (insn->off)
1630 				emit_a32_movsx_r64(is64, insn->off, dst, src, ctx);
1708 		emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code), off);
1727 		emit_udivmod64(rd, rd, rs, ctx, BPF_OP(code), off);
1859 	/* LDX: dst = *(size *)(src + off) */
1864 	/* LDSX: dst = *(signed size *)(src + off) */
1870 			emit_ldsx_r(dst, rn, off, ctx, BPF_SIZE(code));
1872 			emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code));
1877 	/* ST: *(size *)(dst + off) = imm */
1893 		emit_str_r(dst_lo, tmp2, off, ctx, BPF_SIZE(code));
1899 	/* STX: *(size *)(dst + off) = src */
1905 		emit_str_r(dst_lo, rs, off, ctx, BPF_SIZE(code));
1907 	/* PC += off if dst == src */
1908 	/* PC += off if dst > src */
1909 	/* PC += off if dst >= src */
1910 	/* PC += off if dst < src */
1911 	/* PC += off if dst <= src */
1912 	/* PC += off if dst != src */
1913 	/* PC += off if dst > src (signed) */
1914 	/* PC += off if dst >= src (signed) */
1915 	/* PC += off if dst < src (signed) */
1916 	/* PC += off if dst <= src (signed) */
1917 	/* PC += off if dst & src */
1944 	/* PC += off if dst == imm */
1945 	/* PC += off if dst > imm */
1946 	/* PC += off if dst >= imm */
1947 	/* PC += off if dst < imm */
1948 	/* PC += off if dst <= imm */
1949 	/* PC += off if dst != imm */
1950 	/* PC += off if dst > imm (signed) */
1951 	/* PC += off if dst >= imm (signed) */
1952 	/* PC += off if dst < imm (signed) */
1953 	/* PC += off if dst <= imm (signed) */
1954 	/* PC += off if dst & imm */
1977 		if (off == 0)
1992 		jmp_offset = bpf2a32_offset(i+off, i, ctx);
2033 		else if (BPF_CLASS(code) == BPF_JMP && off != 0)
2034 			jmp_offset = bpf2a32_offset(i + off, i, ctx);