Lines Matching refs:ctx
111 static void emit_imm(const s8 rd, s32 imm, struct rv_jit_context *ctx)
117 emit(rv_lui(rd, upper), ctx);
118 emit(rv_addi(rd, rd, lower), ctx);
120 emit(rv_addi(rd, RV_REG_ZERO, lower), ctx);
124 static void emit_imm32(const s8 *rd, s32 imm, struct rv_jit_context *ctx)
127 emit_imm(lo(rd), imm, ctx);
131 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
133 emit(rv_addi(hi(rd), RV_REG_ZERO, -1), ctx);
137 struct rv_jit_context *ctx)
139 emit_imm(lo(rd), imm_lo, ctx);
140 emit_imm(hi(rd), imm_hi, ctx);
143 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
145 int stack_adjust = ctx->stack_size;
150 emit(rv_addi(RV_REG_A0, lo(r0), 0), ctx);
151 emit(rv_addi(RV_REG_A1, hi(r0), 0), ctx);
155 emit(rv_lw(RV_REG_RA, stack_adjust - 4, RV_REG_SP), ctx);
156 emit(rv_lw(RV_REG_FP, stack_adjust - 8, RV_REG_SP), ctx);
157 emit(rv_lw(RV_REG_S1, stack_adjust - 12, RV_REG_SP), ctx);
158 emit(rv_lw(RV_REG_S2, stack_adjust - 16, RV_REG_SP), ctx);
159 emit(rv_lw(RV_REG_S3, stack_adjust - 20, RV_REG_SP), ctx);
160 emit(rv_lw(RV_REG_S4, stack_adjust - 24, RV_REG_SP), ctx);
161 emit(rv_lw(RV_REG_S5, stack_adjust - 28, RV_REG_SP), ctx);
162 emit(rv_lw(RV_REG_S6, stack_adjust - 32, RV_REG_SP), ctx);
163 emit(rv_lw(RV_REG_S7, stack_adjust - 36, RV_REG_SP), ctx);
165 emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx);
174 emit(rv_jalr(RV_REG_ZERO, RV_REG_T0, 4), ctx);
176 emit(rv_jalr(RV_REG_ZERO, RV_REG_RA, 0), ctx);
186 struct rv_jit_context *ctx)
189 emit(rv_lw(hi(tmp), hi(reg), RV_REG_FP), ctx);
190 emit(rv_lw(lo(tmp), lo(reg), RV_REG_FP), ctx);
197 struct rv_jit_context *ctx)
200 emit(rv_sw(RV_REG_FP, hi(reg), hi(src)), ctx);
201 emit(rv_sw(RV_REG_FP, lo(reg), lo(src)), ctx);
206 struct rv_jit_context *ctx)
209 emit(rv_lw(lo(tmp), lo(reg), RV_REG_FP), ctx);
216 struct rv_jit_context *ctx)
219 emit(rv_sw(RV_REG_FP, lo(reg), lo(src)), ctx);
220 if (!ctx->prog->aux->verifier_zext)
221 emit(rv_sw(RV_REG_FP, hi(reg), RV_REG_ZERO), ctx);
222 } else if (!ctx->prog->aux->verifier_zext) {
223 emit(rv_addi(hi(reg), RV_REG_ZERO, 0), ctx);
228 struct rv_jit_context *ctx)
233 emit(rv_jal(rd, rvoff >> 1), ctx);
239 emit(rv_auipc(RV_REG_T1, upper), ctx);
240 emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
244 struct rv_jit_context *ctx, const u8 op)
247 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
251 emit_imm32(rd, imm, ctx);
255 emit(rv_andi(lo(rd), lo(rd), imm), ctx);
257 emit_imm(RV_REG_T0, imm, ctx);
258 emit(rv_and(lo(rd), lo(rd), RV_REG_T0), ctx);
261 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
265 emit(rv_ori(lo(rd), lo(rd), imm), ctx);
267 emit_imm(RV_REG_T0, imm, ctx);
268 emit(rv_or(lo(rd), lo(rd), RV_REG_T0), ctx);
271 emit(rv_ori(hi(rd), RV_REG_ZERO, -1), ctx);
275 emit(rv_xori(lo(rd), lo(rd), imm), ctx);
277 emit_imm(RV_REG_T0, imm, ctx);
278 emit(rv_xor(lo(rd), lo(rd), RV_REG_T0), ctx);
281 emit(rv_xori(hi(rd), hi(rd), -1), ctx);
285 emit(rv_slli(hi(rd), lo(rd), imm - 32), ctx);
286 emit(rv_addi(lo(rd), RV_REG_ZERO, 0), ctx);
290 emit(rv_srli(RV_REG_T0, lo(rd), 32 - imm), ctx);
291 emit(rv_slli(hi(rd), hi(rd), imm), ctx);
292 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
293 emit(rv_slli(lo(rd), lo(rd), imm), ctx);
298 emit(rv_srli(lo(rd), hi(rd), imm - 32), ctx);
299 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
303 emit(rv_slli(RV_REG_T0, hi(rd), 32 - imm), ctx);
304 emit(rv_srli(lo(rd), lo(rd), imm), ctx);
305 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
306 emit(rv_srli(hi(rd), hi(rd), imm), ctx);
311 emit(rv_srai(lo(rd), hi(rd), imm - 32), ctx);
312 emit(rv_srai(hi(rd), hi(rd), 31), ctx);
316 emit(rv_slli(RV_REG_T0, hi(rd), 32 - imm), ctx);
317 emit(rv_srli(lo(rd), lo(rd), imm), ctx);
318 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
319 emit(rv_srai(hi(rd), hi(rd), imm), ctx);
324 bpf_put_reg64(dst, rd, ctx);
328 struct rv_jit_context *ctx, const u8 op)
331 const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
335 emit_imm(lo(rd), imm, ctx);
339 emit(rv_addi(lo(rd), lo(rd), imm), ctx);
341 emit_imm(RV_REG_T0, imm, ctx);
342 emit(rv_add(lo(rd), lo(rd), RV_REG_T0), ctx);
347 emit(rv_addi(lo(rd), lo(rd), -imm), ctx);
349 emit_imm(RV_REG_T0, imm, ctx);
350 emit(rv_sub(lo(rd), lo(rd), RV_REG_T0), ctx);
355 emit(rv_andi(lo(rd), lo(rd), imm), ctx);
357 emit_imm(RV_REG_T0, imm, ctx);
358 emit(rv_and(lo(rd), lo(rd), RV_REG_T0), ctx);
363 emit(rv_ori(lo(rd), lo(rd), imm), ctx);
365 emit_imm(RV_REG_T0, imm, ctx);
366 emit(rv_or(lo(rd), lo(rd), RV_REG_T0), ctx);
371 emit(rv_xori(lo(rd), lo(rd), imm), ctx);
373 emit_imm(RV_REG_T0, imm, ctx);
374 emit(rv_xor(lo(rd), lo(rd), RV_REG_T0), ctx);
379 emit(rv_slli(lo(rd), lo(rd), imm), ctx);
381 emit_imm(RV_REG_T0, imm, ctx);
382 emit(rv_sll(lo(rd), lo(rd), RV_REG_T0), ctx);
387 emit(rv_srli(lo(rd), lo(rd), imm), ctx);
389 emit_imm(RV_REG_T0, imm, ctx);
390 emit(rv_srl(lo(rd), lo(rd), RV_REG_T0), ctx);
395 emit(rv_srai(lo(rd), lo(rd), imm), ctx);
397 emit_imm(RV_REG_T0, imm, ctx);
398 emit(rv_sra(lo(rd), lo(rd), RV_REG_T0), ctx);
403 bpf_put_reg32(dst, rd, ctx);
407 struct rv_jit_context *ctx, const u8 op)
411 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
412 const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
416 emit(rv_addi(lo(rd), lo(rs), 0), ctx);
417 emit(rv_addi(hi(rd), hi(rs), 0), ctx);
421 emit(rv_srli(RV_REG_T0, lo(rd), 31), ctx);
422 emit(rv_slli(hi(rd), hi(rd), 1), ctx);
423 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
424 emit(rv_slli(lo(rd), lo(rd), 1), ctx);
426 emit(rv_add(lo(rd), lo(rd), lo(rs)), ctx);
427 emit(rv_sltu(RV_REG_T0, lo(rd), lo(rs)), ctx);
428 emit(rv_add(hi(rd), hi(rd), hi(rs)), ctx);
429 emit(rv_add(hi(rd), hi(rd), RV_REG_T0), ctx);
433 emit(rv_sub(RV_REG_T1, hi(rd), hi(rs)), ctx);
434 emit(rv_sltu(RV_REG_T0, lo(rd), lo(rs)), ctx);
435 emit(rv_sub(hi(rd), RV_REG_T1, RV_REG_T0), ctx);
436 emit(rv_sub(lo(rd), lo(rd), lo(rs)), ctx);
439 emit(rv_and(lo(rd), lo(rd), lo(rs)), ctx);
440 emit(rv_and(hi(rd), hi(rd), hi(rs)), ctx);
443 emit(rv_or(lo(rd), lo(rd), lo(rs)), ctx);
444 emit(rv_or(hi(rd), hi(rd), hi(rs)), ctx);
447 emit(rv_xor(lo(rd), lo(rd), lo(rs)), ctx);
448 emit(rv_xor(hi(rd), hi(rd), hi(rs)), ctx);
451 emit(rv_mul(RV_REG_T0, hi(rs), lo(rd)), ctx);
452 emit(rv_mul(hi(rd), hi(rd), lo(rs)), ctx);
453 emit(rv_mulhu(RV_REG_T1, lo(rd), lo(rs)), ctx);
454 emit(rv_add(hi(rd), hi(rd), RV_REG_T0), ctx);
455 emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx);
456 emit(rv_add(hi(rd), hi(rd), RV_REG_T1), ctx);
459 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx);
460 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx);
461 emit(rv_sll(hi(rd), lo(rd), RV_REG_T0), ctx);
462 emit(rv_addi(lo(rd), RV_REG_ZERO, 0), ctx);
463 emit(rv_jal(RV_REG_ZERO, 16), ctx);
464 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx);
465 emit(rv_srli(RV_REG_T0, lo(rd), 1), ctx);
466 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx);
467 emit(rv_srl(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx);
468 emit(rv_sll(hi(rd), hi(rd), lo(rs)), ctx);
469 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
470 emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx);
473 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx);
474 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx);
475 emit(rv_srl(lo(rd), hi(rd), RV_REG_T0), ctx);
476 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
477 emit(rv_jal(RV_REG_ZERO, 16), ctx);
478 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx);
479 emit(rv_slli(RV_REG_T0, hi(rd), 1), ctx);
480 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx);
481 emit(rv_sll(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx);
482 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
483 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
484 emit(rv_srl(hi(rd), hi(rd), lo(rs)), ctx);
487 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx);
488 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx);
489 emit(rv_sra(lo(rd), hi(rd), RV_REG_T0), ctx);
490 emit(rv_srai(hi(rd), hi(rd), 31), ctx);
491 emit(rv_jal(RV_REG_ZERO, 16), ctx);
492 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx);
493 emit(rv_slli(RV_REG_T0, hi(rd), 1), ctx);
494 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx);
495 emit(rv_sll(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx);
496 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
497 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
498 emit(rv_sra(hi(rd), hi(rd), lo(rs)), ctx);
501 emit(rv_sub(lo(rd), RV_REG_ZERO, lo(rd)), ctx);
502 emit(rv_sltu(RV_REG_T0, RV_REG_ZERO, lo(rd)), ctx);
503 emit(rv_sub(hi(rd), RV_REG_ZERO, hi(rd)), ctx);
504 emit(rv_sub(hi(rd), hi(rd), RV_REG_T0), ctx);
508 bpf_put_reg64(dst, rd, ctx);
512 struct rv_jit_context *ctx, const u8 op)
516 const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
517 const s8 *rs = bpf_get_reg32(src, tmp2, ctx);
521 emit(rv_addi(lo(rd), lo(rs), 0), ctx);
524 emit(rv_add(lo(rd), lo(rd), lo(rs)), ctx);
527 emit(rv_sub(lo(rd), lo(rd), lo(rs)), ctx);
530 emit(rv_and(lo(rd), lo(rd), lo(rs)), ctx);
533 emit(rv_or(lo(rd), lo(rd), lo(rs)), ctx);
536 emit(rv_xor(lo(rd), lo(rd), lo(rs)), ctx);
539 emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx);
542 emit(rv_divu(lo(rd), lo(rd), lo(rs)), ctx);
545 emit(rv_remu(lo(rd), lo(rd), lo(rs)), ctx);
548 emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx);
551 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
554 emit(rv_sra(lo(rd), lo(rd), lo(rs)), ctx);
557 emit(rv_sub(lo(rd), RV_REG_ZERO, lo(rd)), ctx);
561 bpf_put_reg32(dst, rd, ctx);
565 struct rv_jit_context *ctx, const u8 op)
567 int e, s = ctx->ninsns;
571 const s8 *rs1 = bpf_get_reg64(src1, tmp1, ctx);
572 const s8 *rs2 = bpf_get_reg64(src2, tmp2, ctx);
587 emit(rv_bne(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
588 emit(rv_bne(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
591 emit(rv_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx);
592 emit(rv_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
593 emit(rv_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
596 emit(rv_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx);
597 emit(rv_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
598 emit(rv_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
601 emit(rv_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx);
602 emit(rv_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
603 emit(rv_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
606 emit(rv_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx);
607 emit(rv_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
608 emit(rv_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
611 emit(rv_bne(hi(rs1), hi(rs2), JUMP(1)), ctx);
612 emit(rv_beq(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
615 emit(rv_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx);
616 emit(rv_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
617 emit(rv_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
620 emit(rv_blt(hi(rs1), hi(rs2), JUMP(2)), ctx);
621 emit(rv_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
622 emit(rv_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
625 emit(rv_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx);
626 emit(rv_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
627 emit(rv_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
630 emit(rv_blt(hi(rs1), hi(rs2), JUMP(2)), ctx);
631 emit(rv_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
632 emit(rv_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
635 emit(rv_and(RV_REG_T0, hi(rs1), hi(rs2)), ctx);
636 emit(rv_bne(RV_REG_T0, RV_REG_ZERO, JUMP(2)), ctx);
637 emit(rv_and(RV_REG_T0, lo(rs1), lo(rs2)), ctx);
638 emit(rv_beq(RV_REG_T0, RV_REG_ZERO, NO_JUMP(0)), ctx);
645 e = ctx->ninsns;
648 emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);
652 static int emit_bcc(u8 op, u8 rd, u8 rs, int rvoff, struct rv_jit_context *ctx)
654 int e, s = ctx->ninsns;
678 emit(rv_beq(rd, rs, off), ctx);
681 emit(rv_bgtu(rd, rs, off), ctx);
684 emit(rv_bltu(rd, rs, off), ctx);
687 emit(rv_bgeu(rd, rs, off), ctx);
690 emit(rv_bleu(rd, rs, off), ctx);
693 emit(rv_bne(rd, rs, off), ctx);
696 emit(rv_bgt(rd, rs, off), ctx);
699 emit(rv_blt(rd, rs, off), ctx);
702 emit(rv_bge(rd, rs, off), ctx);
705 emit(rv_ble(rd, rs, off), ctx);
708 emit(rv_and(RV_REG_T0, rd, rs), ctx);
709 emit(rv_beq(RV_REG_T0, RV_REG_ZERO, off), ctx);
714 e = ctx->ninsns;
717 emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);
723 struct rv_jit_context *ctx, const u8 op)
725 int e, s = ctx->ninsns;
729 const s8 *rs1 = bpf_get_reg32(src1, tmp1, ctx);
730 const s8 *rs2 = bpf_get_reg32(src2, tmp2, ctx);
732 e = ctx->ninsns;
736 if (emit_bcc(op, lo(rs1), lo(rs2), rvoff, ctx))
742 static void emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx)
750 emit(rv_addi(RV_REG_SP, RV_REG_SP, -16), ctx);
751 emit(rv_sw(RV_REG_SP, 0, lo(r5)), ctx);
752 emit(rv_sw(RV_REG_SP, 4, hi(r5)), ctx);
755 emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx);
762 emit(rv_lui(RV_REG_T1, upper), ctx);
763 emit(rv_jalr(RV_REG_RA, RV_REG_T1, lower), ctx);
766 emit(rv_addi(RV_REG_TCC, RV_REG_TCC_SAVED, 0), ctx);
769 emit(rv_addi(lo(r0), RV_REG_A0, 0), ctx);
770 emit(rv_addi(hi(r0), RV_REG_A1, 0), ctx);
771 emit(rv_addi(RV_REG_SP, RV_REG_SP, 16), ctx);
774 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
777 * R1 -> &ctx
781 int tc_ninsn, off, start_insn = ctx->ninsns;
785 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
786 ctx->offset[0];
792 emit(rv_lw(RV_REG_T1, off, lo(arr_reg)), ctx);
798 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
799 emit_bcc(BPF_JGE, lo(idx_reg), RV_REG_T1, off, ctx);
805 emit(rv_addi(RV_REG_TCC, RV_REG_TCC, -1), ctx);
806 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
807 emit_bcc(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx);
814 emit(rv_slli(RV_REG_T0, lo(idx_reg), 2), ctx);
815 emit(rv_add(RV_REG_T0, RV_REG_T0, lo(arr_reg)), ctx);
819 emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx);
820 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
821 emit_bcc(BPF_JEQ, RV_REG_T0, RV_REG_ZERO, off, ctx);
830 emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx);
832 __build_epilogue(true, ctx);
837 struct rv_jit_context *ctx, const u8 size)
841 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
842 const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
844 emit_imm(RV_REG_T0, off, ctx);
845 emit(rv_add(RV_REG_T0, RV_REG_T0, lo(rs)), ctx);
849 emit(rv_lbu(lo(rd), 0, RV_REG_T0), ctx);
850 if (!ctx->prog->aux->verifier_zext)
851 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
854 emit(rv_lhu(lo(rd), 0, RV_REG_T0), ctx);
855 if (!ctx->prog->aux->verifier_zext)
856 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
859 emit(rv_lw(lo(rd), 0, RV_REG_T0), ctx);
860 if (!ctx->prog->aux->verifier_zext)
861 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
864 emit(rv_lw(lo(rd), 0, RV_REG_T0), ctx);
865 emit(rv_lw(hi(rd), 4, RV_REG_T0), ctx);
869 bpf_put_reg64(dst, rd, ctx);
874 struct rv_jit_context *ctx, const u8 size,
879 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
880 const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
885 emit_imm(RV_REG_T0, off, ctx);
886 emit(rv_add(RV_REG_T0, RV_REG_T0, lo(rd)), ctx);
890 emit(rv_sb(RV_REG_T0, 0, lo(rs)), ctx);
893 emit(rv_sh(RV_REG_T0, 0, lo(rs)), ctx);
898 emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx);
902 ctx);
907 emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx);
908 emit(rv_sw(RV_REG_T0, 4, hi(rs)), ctx);
915 static void emit_rev16(const s8 rd, struct rv_jit_context *ctx)
917 emit(rv_slli(rd, rd, 16), ctx);
918 emit(rv_slli(RV_REG_T1, rd, 8), ctx);
919 emit(rv_srli(rd, rd, 8), ctx);
920 emit(rv_add(RV_REG_T1, rd, RV_REG_T1), ctx);
921 emit(rv_srli(rd, RV_REG_T1, 16), ctx);
924 static void emit_rev32(const s8 rd, struct rv_jit_context *ctx)
926 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 0), ctx);
927 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
928 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
929 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx);
930 emit(rv_srli(rd, rd, 8), ctx);
931 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
932 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
933 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx);
934 emit(rv_srli(rd, rd, 8), ctx);
935 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
936 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
937 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx);
938 emit(rv_srli(rd, rd, 8), ctx);
939 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
940 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
941 emit(rv_addi(rd, RV_REG_T1, 0), ctx);
944 static void emit_zext64(const s8 *dst, struct rv_jit_context *ctx)
949 rd = bpf_get_reg64(dst, tmp1, ctx);
950 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
951 bpf_put_reg64(dst, rd, ctx);
954 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
959 int s, e, rvoff, i = insn - ctx->prog->insnsi;
989 emit_imm32(tmp2, imm, ctx);
992 emit_alu_r64(dst, src, ctx, BPF_OP(code));
996 emit_alu_r64(dst, tmp2, ctx, BPF_OP(code));
1012 emit_alu_i64(dst, imm, ctx, BPF_OP(code));
1018 emit_zext64(dst, ctx);
1042 emit_imm32(tmp2, imm, ctx);
1045 emit_alu_r32(dst, src, ctx, BPF_OP(code));
1061 emit_alu_i32(dst, imm, ctx, BPF_OP(code));
1069 emit_alu_r32(dst, tmp2, ctx, BPF_OP(code));
1074 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1078 emit(rv_slli(lo(rd), lo(rd), 16), ctx);
1079 emit(rv_srli(lo(rd), lo(rd), 16), ctx);
1082 if (!ctx->prog->aux->verifier_zext)
1083 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1093 bpf_put_reg64(dst, rd, ctx);
1099 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1103 emit_rev16(lo(rd), ctx);
1104 if (!ctx->prog->aux->verifier_zext)
1105 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1108 emit_rev32(lo(rd), ctx);
1109 if (!ctx->prog->aux->verifier_zext)
1110 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1114 emit(rv_addi(RV_REG_T0, lo(rd), 0), ctx);
1115 emit(rv_addi(lo(rd), hi(rd), 0), ctx);
1116 emit(rv_addi(hi(rd), RV_REG_T0, 0), ctx);
1119 emit_rev32(lo(rd), ctx);
1120 emit_rev32(hi(rd), ctx);
1127 bpf_put_reg64(dst, rd, ctx);
1132 rvoff = rv_offset(i, off, ctx);
1133 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
1142 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr,
1146 emit_call(fixed, addr, ctx);
1151 if (emit_bpf_tail_call(i, ctx))
1209 rvoff = rv_offset(i, off, ctx);
1211 s = ctx->ninsns;
1212 emit_imm32(tmp2, imm, ctx);
1214 e = ctx->ninsns;
1219 emit_branch_r64(dst, src, rvoff, ctx, BPF_OP(code));
1221 emit_branch_r32(dst, src, rvoff, ctx, BPF_OP(code));
1225 if (i == ctx->prog->len - 1)
1228 rvoff = epilogue_offset(ctx);
1229 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
1237 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1239 emit_imm64(rd, imm_hi, imm_lo, ctx);
1240 bpf_put_reg64(dst, rd, ctx);
1248 if (emit_load_r64(dst, src, off, ctx, BPF_SIZE(code)))
1266 emit_imm32(tmp2, imm, ctx);
1270 if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code),
1283 if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code),
1304 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog)
1310 round_up(ctx->prog->aux->stack_depth, STACK_ALIGN);
1325 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1327 emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx);
1330 emit(rv_sw(RV_REG_SP, stack_adjust - 4, RV_REG_RA), ctx);
1331 emit(rv_sw(RV_REG_SP, stack_adjust - 8, RV_REG_FP), ctx);
1332 emit(rv_sw(RV_REG_SP, stack_adjust - 12, RV_REG_S1), ctx);
1333 emit(rv_sw(RV_REG_SP, stack_adjust - 16, RV_REG_S2), ctx);
1334 emit(rv_sw(RV_REG_SP, stack_adjust - 20, RV_REG_S3), ctx);
1335 emit(rv_sw(RV_REG_SP, stack_adjust - 24, RV_REG_S4), ctx);
1336 emit(rv_sw(RV_REG_SP, stack_adjust - 28, RV_REG_S5), ctx);
1337 emit(rv_sw(RV_REG_SP, stack_adjust - 32, RV_REG_S6), ctx);
1338 emit(rv_sw(RV_REG_SP, stack_adjust - 36, RV_REG_S7), ctx);
1341 emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx);
1344 emit(rv_addi(lo(fp), RV_REG_SP, bpf_stack_adjust), ctx);
1345 emit(rv_addi(hi(fp), RV_REG_ZERO, 0), ctx);
1348 emit(rv_addi(lo(r1), RV_REG_A0, 0), ctx);
1349 emit(rv_addi(hi(r1), RV_REG_ZERO, 0), ctx);
1351 ctx->stack_size = stack_adjust;
1354 void bpf_jit_build_epilogue(struct rv_jit_context *ctx)
1356 __build_epilogue(false, ctx);