riscv/net/bpf_jit_comp32.c

111 static void emit_imm(const s8 rd, s32 imm, struct rv_jit_context *ctx)
117 		emit(rv_lui(rd, upper), ctx);
118 		emit(rv_addi(rd, rd, lower), ctx);
120 		emit(rv_addi(rd, RV_REG_ZERO, lower), ctx);
124 static void emit_imm32(const s8 *rd, s32 imm, struct rv_jit_context *ctx)
127 	emit_imm(lo(rd), imm, ctx);
131 		emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
133 		emit(rv_addi(hi(rd), RV_REG_ZERO, -1), ctx);
136 static void emit_imm64(const s8 *rd, s32 imm_hi, s32 imm_lo,
139 	emit_imm(lo(rd), imm_lo, ctx);
140 	emit_imm(hi(rd), imm_hi, ctx);
227 static void emit_jump_and_link(u8 rd, s32 rvoff, bool force_jalr,
233 		emit(rv_jal(rd, rvoff >> 1), ctx);
240 	emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
247 	const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
251 		emit_imm32(rd, imm, ctx);
255 			emit(rv_andi(lo(rd), lo(rd), imm), ctx);
258 			emit(rv_and(lo(rd), lo(rd), RV_REG_T0), ctx);
261 			emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
265 			emit(rv_ori(lo(rd), lo(rd), imm), ctx);
268 			emit(rv_or(lo(rd), lo(rd), RV_REG_T0), ctx);
271 			emit(rv_ori(hi(rd), RV_REG_ZERO, -1), ctx);
275 			emit(rv_xori(lo(rd), lo(rd), imm), ctx);
278 			emit(rv_xor(lo(rd), lo(rd), RV_REG_T0), ctx);
281 			emit(rv_xori(hi(rd), hi(rd), -1), ctx);
285 			emit(rv_slli(hi(rd), lo(rd), imm - 32), ctx);
286 			emit(rv_addi(lo(rd), RV_REG_ZERO, 0), ctx);
290 			emit(rv_srli(RV_REG_T0, lo(rd), 32 - imm), ctx);
291 			emit(rv_slli(hi(rd), hi(rd), imm), ctx);
292 			emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
293 			emit(rv_slli(lo(rd), lo(rd), imm), ctx);
298 			emit(rv_srli(lo(rd), hi(rd), imm - 32), ctx);
299 			emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
303 			emit(rv_slli(RV_REG_T0, hi(rd), 32 - imm), ctx);
304 			emit(rv_srli(lo(rd), lo(rd), imm), ctx);
305 			emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
306 			emit(rv_srli(hi(rd), hi(rd), imm), ctx);
311 			emit(rv_srai(lo(rd), hi(rd), imm - 32), ctx);
312 			emit(rv_srai(hi(rd), hi(rd), 31), ctx);
316 			emit(rv_slli(RV_REG_T0, hi(rd), 32 - imm), ctx);
317 			emit(rv_srli(lo(rd), lo(rd), imm), ctx);
318 			emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
319 			emit(rv_srai(hi(rd), hi(rd), imm), ctx);
324 	bpf_put_reg64(dst, rd, ctx);
331 	const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
335 		emit_imm(lo(rd), imm, ctx);
339 			emit(rv_addi(lo(rd), lo(rd), imm), ctx);
342 			emit(rv_add(lo(rd), lo(rd), RV_REG_T0), ctx);
347 			emit(rv_addi(lo(rd), lo(rd), -imm), ctx);
350 			emit(rv_sub(lo(rd), lo(rd), RV_REG_T0), ctx);
355 			emit(rv_andi(lo(rd), lo(rd), imm), ctx);
358 			emit(rv_and(lo(rd), lo(rd), RV_REG_T0), ctx);
363 			emit(rv_ori(lo(rd), lo(rd), imm), ctx);
366 			emit(rv_or(lo(rd), lo(rd), RV_REG_T0), ctx);
371 			emit(rv_xori(lo(rd), lo(rd), imm), ctx);
374 			emit(rv_xor(lo(rd), lo(rd), RV_REG_T0), ctx);
379 			emit(rv_slli(lo(rd), lo(rd), imm), ctx);
382 			emit(rv_sll(lo(rd), lo(rd), RV_REG_T0), ctx);
387 			emit(rv_srli(lo(rd), lo(rd), imm), ctx);
390 			emit(rv_srl(lo(rd), lo(rd), RV_REG_T0), ctx);
395 			emit(rv_srai(lo(rd), lo(rd), imm), ctx);
398 			emit(rv_sra(lo(rd), lo(rd), RV_REG_T0), ctx);
403 	bpf_put_reg32(dst, rd, ctx);
411 	const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
416 		emit(rv_addi(lo(rd), lo(rs), 0), ctx);
417 		emit(rv_addi(hi(rd), hi(rs), 0), ctx);
420 		if (rd == rs) {
421 			emit(rv_srli(RV_REG_T0, lo(rd), 31), ctx);
422 			emit(rv_slli(hi(rd), hi(rd), 1), ctx);
423 			emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
424 			emit(rv_slli(lo(rd), lo(rd), 1), ctx);
426 			emit(rv_add(lo(rd), lo(rd), lo(rs)), ctx);
427 			emit(rv_sltu(RV_REG_T0, lo(rd), lo(rs)), ctx);
428 			emit(rv_add(hi(rd), hi(rd), hi(rs)), ctx);
429 			emit(rv_add(hi(rd), hi(rd), RV_REG_T0), ctx);
433 		emit(rv_sub(RV_REG_T1, hi(rd), hi(rs)), ctx);
434 		emit(rv_sltu(RV_REG_T0, lo(rd), lo(rs)), ctx);
435 		emit(rv_sub(hi(rd), RV_REG_T1, RV_REG_T0), ctx);
436 		emit(rv_sub(lo(rd), lo(rd), lo(rs)), ctx);
439 		emit(rv_and(lo(rd), lo(rd), lo(rs)), ctx);
440 		emit(rv_and(hi(rd), hi(rd), hi(rs)), ctx);
443 		emit(rv_or(lo(rd), lo(rd), lo(rs)), ctx);
444 		emit(rv_or(hi(rd), hi(rd), hi(rs)), ctx);
447 		emit(rv_xor(lo(rd), lo(rd), lo(rs)), ctx);
448 		emit(rv_xor(hi(rd), hi(rd), hi(rs)), ctx);
451 		emit(rv_mul(RV_REG_T0, hi(rs), lo(rd)), ctx);
452 		emit(rv_mul(hi(rd), hi(rd), lo(rs)), ctx);
453 		emit(rv_mulhu(RV_REG_T1, lo(rd), lo(rs)), ctx);
454 		emit(rv_add(hi(rd), hi(rd), RV_REG_T0), ctx);
455 		emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx);
456 		emit(rv_add(hi(rd), hi(rd), RV_REG_T1), ctx);
461 		emit(rv_sll(hi(rd), lo(rd), RV_REG_T0), ctx);
462 		emit(rv_addi(lo(rd), RV_REG_ZERO, 0), ctx);
465 		emit(rv_srli(RV_REG_T0, lo(rd), 1), ctx);
468 		emit(rv_sll(hi(rd), hi(rd), lo(rs)), ctx);
469 		emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
470 		emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx);
475 		emit(rv_srl(lo(rd), hi(rd), RV_REG_T0), ctx);
476 		emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
479 		emit(rv_slli(RV_REG_T0, hi(rd), 1), ctx);
482 		emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
483 		emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
484 		emit(rv_srl(hi(rd), hi(rd), lo(rs)), ctx);
489 		emit(rv_sra(lo(rd), hi(rd), RV_REG_T0), ctx);
490 		emit(rv_srai(hi(rd), hi(rd), 31), ctx);
493 		emit(rv_slli(RV_REG_T0, hi(rd), 1), ctx);
496 		emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
497 		emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
498 		emit(rv_sra(hi(rd), hi(rd), lo(rs)), ctx);
501 		emit(rv_sub(lo(rd), RV_REG_ZERO, lo(rd)), ctx);
502 		emit(rv_sltu(RV_REG_T0, RV_REG_ZERO, lo(rd)), ctx);
503 		emit(rv_sub(hi(rd), RV_REG_ZERO, hi(rd)), ctx);
504 		emit(rv_sub(hi(rd), hi(rd), RV_REG_T0), ctx);
508 	bpf_put_reg64(dst, rd, ctx);
516 	const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
521 		emit(rv_addi(lo(rd), lo(rs), 0), ctx);
524 		emit(rv_add(lo(rd), lo(rd), lo(rs)), ctx);
527 		emit(rv_sub(lo(rd), lo(rd), lo(rs)), ctx);
530 		emit(rv_and(lo(rd), lo(rd), lo(rs)), ctx);
533 		emit(rv_or(lo(rd), lo(rd), lo(rs)), ctx);
536 		emit(rv_xor(lo(rd), lo(rd), lo(rs)), ctx);
539 		emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx);
542 		emit(rv_divu(lo(rd), lo(rd), lo(rs)), ctx);
545 		emit(rv_remu(lo(rd), lo(rd), lo(rs)), ctx);
548 		emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx);
551 		emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
554 		emit(rv_sra(lo(rd), lo(rd), lo(rs)), ctx);
557 		emit(rv_sub(lo(rd), RV_REG_ZERO, lo(rd)), ctx);
561 	bpf_put_reg32(dst, rd, ctx);
652 static int emit_bcc(u8 op, u8 rd, u8 rs, int rvoff, struct rv_jit_context *ctx)
678 		emit(rv_beq(rd, rs, off), ctx);
681 		emit(rv_bgtu(rd, rs, off), ctx);
684 		emit(rv_bltu(rd, rs, off), ctx);
687 		emit(rv_bgeu(rd, rs, off), ctx);
690 		emit(rv_bleu(rd, rs, off), ctx);
693 		emit(rv_bne(rd, rs, off), ctx);
696 		emit(rv_bgt(rd, rs, off), ctx);
699 		emit(rv_blt(rd, rs, off), ctx);
702 		emit(rv_bge(rd, rs, off), ctx);
705 		emit(rv_ble(rd, rs, off), ctx);
708 		emit(rv_and(RV_REG_T0, rd, rs), ctx);
841 	const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
849 		emit(rv_lbu(lo(rd), 0, RV_REG_T0), ctx);
851 			emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
854 		emit(rv_lhu(lo(rd), 0, RV_REG_T0), ctx);
856 			emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
859 		emit(rv_lw(lo(rd), 0, RV_REG_T0), ctx);
861 			emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
864 		emit(rv_lw(lo(rd), 0, RV_REG_T0), ctx);
865 		emit(rv_lw(hi(rd), 4, RV_REG_T0), ctx);
869 	bpf_put_reg64(dst, rd, ctx);
879 	const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
886 	emit(rv_add(RV_REG_T0, RV_REG_T0, lo(rd)), ctx);
915 static void emit_rev16(const s8 rd, struct rv_jit_context *ctx)
917 	emit(rv_slli(rd, rd, 16), ctx);
918 	emit(rv_slli(RV_REG_T1, rd, 8), ctx);
919 	emit(rv_srli(rd, rd, 8), ctx);
920 	emit(rv_add(RV_REG_T1, rd, RV_REG_T1), ctx);
921 	emit(rv_srli(rd, RV_REG_T1, 16), ctx);
924 static void emit_rev32(const s8 rd, struct rv_jit_context *ctx)
927 	emit(rv_andi(RV_REG_T0, rd, 255), ctx);
930 	emit(rv_srli(rd, rd, 8), ctx);
931 	emit(rv_andi(RV_REG_T0, rd, 255), ctx);
934 	emit(rv_srli(rd, rd, 8), ctx);
935 	emit(rv_andi(RV_REG_T0, rd, 255), ctx);
938 	emit(rv_srli(rd, rd, 8), ctx);
939 	emit(rv_andi(RV_REG_T0, rd, 255), ctx);
941 	emit(rv_addi(rd, RV_REG_T1, 0), ctx);
946 	const s8 *rd;
949 	rd = bpf_get_reg64(dst, tmp1, ctx);
950 	emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
951 	bpf_put_reg64(dst, rd, ctx);
1074 		const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1078 			emit(rv_slli(lo(rd), lo(rd), 16), ctx);
1079 			emit(rv_srli(lo(rd), lo(rd), 16), ctx);
1083 				emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1093 		bpf_put_reg64(dst, rd, ctx);
1099 		const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1103 			emit_rev16(lo(rd), ctx);
1105 				emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1108 			emit_rev32(lo(rd), ctx);
1110 				emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1114 			emit(rv_addi(RV_REG_T0, lo(rd), 0), ctx);
1115 			emit(rv_addi(lo(rd), hi(rd), 0), ctx);
1116 			emit(rv_addi(hi(rd), RV_REG_T0, 0), ctx);
1119 			emit_rev32(lo(rd), ctx);
1120 			emit_rev32(hi(rd), ctx);
1127 		bpf_put_reg64(dst, rd, ctx);
1237 		const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1239 		emit_imm64(rd, imm_hi, imm_lo, ctx);
1240 		bpf_put_reg64(dst, rd, ctx);