Lines Matching refs:ctx
167 static void clobber_reg64(struct jit_context *ctx, const u8 reg[])
169 clobber_reg(ctx, reg[0]);
170 clobber_reg(ctx, reg[1]);
174 static void emit_mov_se_i64(struct jit_context *ctx, const u8 dst[], s32 imm)
176 emit_mov_i(ctx, lo(dst), imm);
178 emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1);
180 emit(ctx, move, hi(dst), MIPS_R_ZERO);
181 clobber_reg64(ctx, dst);
185 static void emit_zext_ver(struct jit_context *ctx, const u8 dst[])
187 if (!ctx->program->aux->verifier_zext) {
188 emit(ctx, move, hi(dst), MIPS_R_ZERO);
189 clobber_reg(ctx, hi(dst));
194 static void emit_load_delay(struct jit_context *ctx)
197 emit(ctx, nop);
201 static void emit_alu_i64(struct jit_context *ctx,
223 emit_mov_i(ctx, src, imm);
228 emit(ctx, addu, lo(dst), lo(dst), src);
229 emit(ctx, sltu, MIPS_R_T9, lo(dst), src);
230 emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9);
232 emit(ctx, addiu, hi(dst), hi(dst), -1);
236 emit(ctx, sltu, MIPS_R_T9, lo(dst), src);
237 emit(ctx, subu, lo(dst), lo(dst), src);
238 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9);
240 emit(ctx, addiu, hi(dst), hi(dst), 1);
244 emit(ctx, or, lo(dst), lo(dst), src);
246 emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1);
250 emit(ctx, and, lo(dst), lo(dst), src);
252 emit(ctx, move, hi(dst), MIPS_R_ZERO);
256 emit(ctx, xor, lo(dst), lo(dst), src);
258 emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst));
259 emit(ctx, addiu, hi(dst), hi(dst), -1);
263 clobber_reg64(ctx, dst);
267 static void emit_alu_r64(struct jit_context *ctx,
274 emit(ctx, srl, MIPS_R_T9, lo(dst), 31);
275 emit(ctx, addu, lo(dst), lo(dst), lo(dst));
277 emit(ctx, addu, lo(dst), lo(dst), lo(src));
278 emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src));
280 emit(ctx, addu, hi(dst), hi(dst), hi(src));
281 emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9);
285 emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src));
286 emit(ctx, subu, lo(dst), lo(dst), lo(src));
287 emit(ctx, subu, hi(dst), hi(dst), hi(src));
288 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9);
292 emit(ctx, or, lo(dst), lo(dst), lo(src));
293 emit(ctx, or, hi(dst), hi(dst), hi(src));
297 emit(ctx, and, lo(dst), lo(dst), lo(src));
298 emit(ctx, and, hi(dst), hi(dst), hi(src));
302 emit(ctx, xor, lo(dst), lo(dst), lo(src));
303 emit(ctx, xor, hi(dst), hi(dst), hi(src));
306 clobber_reg64(ctx, dst);
310 static void emit_neg_i64(struct jit_context *ctx, const u8 dst[])
312 emit(ctx, sltu, MIPS_R_T9, MIPS_R_ZERO, lo(dst));
313 emit(ctx, subu, lo(dst), MIPS_R_ZERO, lo(dst));
314 emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst));
315 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9);
317 clobber_reg64(ctx, dst);
321 static void emit_shift_i64(struct jit_context *ctx,
328 emit(ctx, srl, MIPS_R_T9, lo(dst), 32 - imm);
329 emit(ctx, sll, lo(dst), lo(dst), imm);
330 emit(ctx, sll, hi(dst), hi(dst), imm);
331 emit(ctx, or, hi(dst), hi(dst), MIPS_R_T9);
333 emit(ctx, sll, hi(dst), lo(dst), imm - 32);
334 emit(ctx, move, lo(dst), MIPS_R_ZERO);
340 emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm);
341 emit(ctx, srl, lo(dst), lo(dst), imm);
342 emit(ctx, srl, hi(dst), hi(dst), imm);
343 emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9);
345 emit(ctx, srl, lo(dst), hi(dst), imm - 32);
346 emit(ctx, move, hi(dst), MIPS_R_ZERO);
352 emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm);
353 emit(ctx, srl, lo(dst), lo(dst), imm);
354 emit(ctx, sra, hi(dst), hi(dst), imm);
355 emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9);
357 emit(ctx, sra, lo(dst), hi(dst), imm - 32);
358 emit(ctx, sra, hi(dst), hi(dst), 31);
362 clobber_reg64(ctx, dst);
366 static void emit_shift_r64(struct jit_context *ctx,
372 emit(ctx, andi, t1, src, 32); /* t1 = src & 32 */
373 emit(ctx, beqz, t1, 16); /* PC += 16 if t1 == 0 */
374 emit(ctx, nor, t2, src, MIPS_R_ZERO); /* t2 = ~src (delay slot) */
380 emit(ctx, sllv, hi(dst), lo(dst), src); /* dh = dl << src */
381 emit(ctx, move, lo(dst), MIPS_R_ZERO); /* dl = 0 */
382 emit(ctx, b, 20); /* PC += 20 */
384 emit(ctx, srl, t1, lo(dst), 1); /* t1 = dl >> 1 */
385 emit(ctx, srlv, t1, t1, t2); /* t1 = t1 >> t2 */
386 emit(ctx, sllv, lo(dst), lo(dst), src); /* dl = dl << src */
387 emit(ctx, sllv, hi(dst), hi(dst), src); /* dh = dh << src */
388 emit(ctx, or, hi(dst), hi(dst), t1); /* dh = dh | t1 */
393 emit(ctx, srlv, lo(dst), hi(dst), src); /* dl = dh >> src */
394 emit(ctx, move, hi(dst), MIPS_R_ZERO); /* dh = 0 */
395 emit(ctx, b, 20); /* PC += 20 */
397 emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */
398 emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */
399 emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >> src */
400 emit(ctx, srlv, hi(dst), hi(dst), src); /* dh = dh >> src */
401 emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */
406 emit(ctx, srav, lo(dst), hi(dst), src); /* dl = dh >>a src */
407 emit(ctx, sra, hi(dst), hi(dst), 31); /* dh = dh >>a 31 */
408 emit(ctx, b, 20); /* PC += 20 */
410 emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */
411 emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */
412 emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >>a src */
413 emit(ctx, srav, hi(dst), hi(dst), src); /* dh = dh >> src */
414 emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */
419 clobber_reg64(ctx, dst);
423 static void emit_mul_i64(struct jit_context *ctx, const u8 dst[], s32 imm)
434 emit_neg_i64(ctx, dst);
437 emit_mov_r(ctx, lo(dst), MIPS_R_ZERO);
438 emit_mov_r(ctx, hi(dst), MIPS_R_ZERO);
443 emit_mov_i(ctx, src, imm);
445 emit(ctx, mul, hi(dst), hi(dst), src);
447 emit(ctx, multu, hi(dst), src);
448 emit(ctx, mflo, hi(dst));
453 emit(ctx, subu, hi(dst), hi(dst), lo(dst));
458 emit(ctx, muhu, tmp, lo(dst), src);
459 emit(ctx, mulu, lo(dst), lo(dst), src);
461 emit(ctx, multu, lo(dst), src);
462 emit(ctx, mflo, lo(dst));
463 emit(ctx, mfhi, tmp);
467 emit(ctx, addu, hi(dst), hi(dst), tmp);
468 clobber_reg64(ctx, dst);
474 static void emit_mul_r64(struct jit_context *ctx,
482 emit(ctx, mul, acc, hi(dst), lo(src));
484 emit(ctx, multu, hi(dst), lo(src));
485 emit(ctx, mflo, acc);
490 emit(ctx, mul, tmp, lo(dst), hi(src));
492 emit(ctx, multu, lo(dst), hi(src));
493 emit(ctx, mflo, tmp);
497 emit(ctx, addu, acc, acc, tmp);
502 emit(ctx, muhu, tmp, lo(dst), lo(src));
503 emit(ctx, mulu, lo(dst), lo(dst), lo(src));
505 emit(ctx, multu, lo(dst), lo(src));
506 emit(ctx, mflo, lo(dst));
507 emit(ctx, mfhi, tmp);
511 emit(ctx, addu, hi(dst), acc, tmp);
512 clobber_reg64(ctx, dst);
525 static void emit_divmod_r64(struct jit_context *ctx,
535 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
540 emit(ctx, move, MIPS_R_T9, src[k]);
541 emit(ctx, move, r1[k], dst[k]);
542 emit(ctx, move, r2[k], MIPS_R_T9);
556 emit_mov_i(ctx, MIPS_R_T9, addr);
557 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
558 emit(ctx, nop); /* Delay slot */
561 emit(ctx, move, dst[0], r0[0]);
562 emit(ctx, move, dst[1], r0[1]);
566 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
568 emit_load_delay(ctx);
570 clobber_reg64(ctx, dst);
571 clobber_reg(ctx, MIPS_R_V0);
572 clobber_reg(ctx, MIPS_R_V1);
573 clobber_reg(ctx, MIPS_R_RA);
577 static void emit_swap8_r(struct jit_context *ctx, u8 dst, u8 src, u8 mask)
581 emit(ctx, and, tmp, src, mask); /* tmp = src & 0x00ff00ff */
582 emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */
583 emit(ctx, srl, dst, src, 8); /* dst = src >> 8 */
584 emit(ctx, and, dst, dst, mask); /* dst = dst & 0x00ff00ff */
585 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */
589 static void emit_swap16_r(struct jit_context *ctx, u8 dst, u8 src)
593 emit(ctx, sll, tmp, src, 16); /* tmp = src << 16 */
594 emit(ctx, srl, dst, src, 16); /* dst = src >> 16 */
595 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */
599 static void emit_bswap_r64(struct jit_context *ctx, const u8 dst[], u32 width)
607 emit(ctx, rotr, tmp, hi(dst), 16);
608 emit(ctx, rotr, hi(dst), lo(dst), 16);
609 emit(ctx, wsbh, lo(dst), tmp);
610 emit(ctx, wsbh, hi(dst), hi(dst));
612 emit_swap16_r(ctx, tmp, lo(dst));
613 emit_swap16_r(ctx, lo(dst), hi(dst));
614 emit(ctx, move, hi(dst), tmp);
616 emit(ctx, lui, tmp, 0xff); /* tmp = 0x00ff0000 */
617 emit(ctx, ori, tmp, tmp, 0xff); /* tmp = 0x00ff00ff */
618 emit_swap8_r(ctx, lo(dst), lo(dst), tmp);
619 emit_swap8_r(ctx, hi(dst), hi(dst), tmp);
626 emit_bswap_r(ctx, lo(dst), width);
627 emit(ctx, move, hi(dst), MIPS_R_ZERO);
630 clobber_reg64(ctx, dst);
634 static void emit_trunc_r64(struct jit_context *ctx, const u8 dst[], u32 width)
641 emit(ctx, move, hi(dst), MIPS_R_ZERO);
642 clobber_reg(ctx, hi(dst));
646 emit(ctx, move, hi(dst), MIPS_R_ZERO);
647 emit(ctx, andi, lo(dst), lo(dst), 0xffff);
648 clobber_reg64(ctx, dst);
654 static void emit_ldx(struct jit_context *ctx,
660 emit(ctx, lbu, lo(dst), off, src);
661 emit(ctx, move, hi(dst), MIPS_R_ZERO);
665 emit(ctx, lhu, lo(dst), off, src);
666 emit(ctx, move, hi(dst), MIPS_R_ZERO);
670 emit(ctx, lw, lo(dst), off, src);
671 emit(ctx, move, hi(dst), MIPS_R_ZERO);
676 emit(ctx, lw, dst[0], off + 4, src);
677 emit(ctx, lw, dst[1], off, src);
679 emit(ctx, lw, dst[1], off, src);
680 emit(ctx, lw, dst[0], off + 4, src);
682 emit_load_delay(ctx);
685 clobber_reg64(ctx, dst);
689 static void emit_stx(struct jit_context *ctx,
695 emit(ctx, sb, lo(src), off, dst);
699 emit(ctx, sh, lo(src), off, dst);
703 emit(ctx, sw, lo(src), off, dst);
707 emit(ctx, sw, src[1], off, dst);
708 emit(ctx, sw, src[0], off + 4, dst);
714 static void emit_atomic_r32(struct jit_context *ctx,
721 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
727 emit(ctx, move, MIPS_R_T9, dst);
729 emit(ctx, move, MIPS_R_A1, src);
730 emit(ctx, addiu, MIPS_R_A0, MIPS_R_T9, off);
732 emit(ctx, move, MIPS_R_A0, src);
733 emit(ctx, addiu, MIPS_R_A1, MIPS_R_T9, off);
772 emit_mov_i(ctx, MIPS_R_T9, addr);
773 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
774 emit(ctx, nop); /* Delay slot */
778 emit(ctx, move, src, MIPS_R_V0);
780 clobber_reg(ctx, src);
784 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
786 emit_load_delay(ctx);
787 clobber_reg(ctx, MIPS_R_RA);
797 static void emit_atomic_r64(struct jit_context *ctx,
806 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
812 emit(ctx, move, MIPS_R_T9, dst);
813 emit(ctx, move, r1[0], src[0]);
814 emit(ctx, move, r1[1], src[1]);
815 emit(ctx, addiu, MIPS_R_A2, MIPS_R_T9, off);
853 emit_mov_i(ctx, MIPS_R_T9, addr);
854 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
855 emit(ctx, nop); /* Delay slot */
859 emit(ctx, move, lo(src), lo(r0));
860 emit(ctx, move, hi(src), hi(r0));
862 clobber_reg64(ctx, src);
866 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
868 emit_load_delay(ctx);
869 clobber_reg(ctx, MIPS_R_RA);
873 static void emit_cmpxchg_r32(struct jit_context *ctx, u8 dst, u8 src, s16 off)
878 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
885 emit(ctx, addiu, MIPS_R_T9, dst, off);
886 emit(ctx, move, MIPS_R_T8, src);
887 emit(ctx, move, MIPS_R_A1, lo(r0));
888 emit(ctx, move, MIPS_R_A0, MIPS_R_T9);
889 emit(ctx, move, MIPS_R_A2, MIPS_R_T8);
892 emit_mov_i(ctx, MIPS_R_T9, (u32)&atomic_cmpxchg);
893 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
894 emit(ctx, nop); /* Delay slot */
897 emit(ctx, move, lo(r0), MIPS_R_V0);
900 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
902 emit_load_delay(ctx);
903 clobber_reg(ctx, MIPS_R_V0);
904 clobber_reg(ctx, MIPS_R_V1);
905 clobber_reg(ctx, MIPS_R_RA);
909 static void emit_cmpxchg_r64(struct jit_context *ctx,
916 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
923 push_regs(ctx, BIT(src[0]) | BIT(src[1]), 0, JIT_RESERVED_STACK);
924 emit(ctx, addiu, MIPS_R_T9, dst, off);
925 emit(ctx, move, r2[0], r0[0]);
926 emit(ctx, move, r2[1], r0[1]);
927 emit(ctx, move, MIPS_R_A0, MIPS_R_T9);
930 emit_mov_i(ctx, MIPS_R_T9, (u32)&atomic64_cmpxchg);
931 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
932 emit(ctx, nop); /* Delay slot */
935 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS,
937 emit_load_delay(ctx);
938 clobber_reg(ctx, MIPS_R_V0);
939 clobber_reg(ctx, MIPS_R_V1);
940 clobber_reg(ctx, MIPS_R_RA);
947 static void emit_movz_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt)
950 emit(ctx, movz, rd, rs, rt); /* rd = rt ? rd : rs */
953 emit(ctx, seleqz, rs, rs, rt); /* rs = 0 if rt == 0 */
954 emit(ctx, selnez, rd, rd, rt); /* rd = 0 if rt != 0 */
956 emit(ctx, or, rd, rd, rs); /* rd = rd | rs */
958 emit(ctx, bnez, rt, 8); /* PC += 8 if rd != 0 */
959 emit(ctx, nop); /* +0: delay slot */
960 emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */
962 clobber_reg(ctx, rd);
963 clobber_reg(ctx, rs);
970 static void emit_movn_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt)
973 emit(ctx, movn, rd, rs, rt); /* rd = rt ? rs : rd */
976 emit(ctx, selnez, rs, rs, rt); /* rs = 0 if rt == 0 */
977 emit(ctx, seleqz, rd, rd, rt); /* rd = 0 if rt != 0 */
979 emit(ctx, or, rd, rd, rs); /* rd = rd | rs */
981 emit(ctx, beqz, rt, 8); /* PC += 8 if rd == 0 */
982 emit(ctx, nop); /* +0: delay slot */
983 emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */
985 clobber_reg(ctx, rd);
986 clobber_reg(ctx, rs);
990 static void emit_sltiu_r64(struct jit_context *ctx, u8 rd,
996 emit_mov_i(ctx, rd, imm); /* rd = imm */
997 emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */
998 emit(ctx, sltiu, tmp, hi(rs), -1); /* tmp = rsh < ~0U */
999 emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */
1002 emit_mov_i(ctx, rd, (s32)imm); /* rd = imm */
1003 emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */
1005 emit(ctx, sltiu, rd, lo(rs), imm); /* rd = rsl < imm */
1007 emit_movn_r(ctx, rd, MIPS_R_ZERO, hi(rs)); /* rd = 0 if rsh */
1012 static void emit_sltu_r64(struct jit_context *ctx, u8 rd,
1017 emit(ctx, sltu, rd, lo(rs), lo(rt)); /* rd = rsl < rtl */
1018 emit(ctx, subu, tmp, hi(rs), hi(rt)); /* tmp = rsh - rth */
1019 emit_movn_r(ctx, rd, MIPS_R_ZERO, tmp); /* rd = 0 if tmp != 0 */
1020 emit(ctx, sltu, tmp, hi(rs), hi(rt)); /* tmp = rsh < rth */
1021 emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */
1025 static void emit_slti_r64(struct jit_context *ctx, u8 rd,
1036 emit_mov_i(ctx, rd, (s32)imm);
1037 emit(ctx, sltu, t1, lo(rs), rd); /* t1 = rsl <u imm */
1038 emit(ctx, sltu, t2, rd, lo(rs)); /* t2 = imm <u rsl */
1039 emit(ctx, srl, rd, hi(rs), 31); /* rd = rsh >> 31 */
1041 emit_movz_r(ctx, t1, t2, rd); /* t1 = rd ? t1 : t2 */
1043 emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */
1050 emit(ctx, addiu, rd, hi(rs), 1); /* rd = rsh + 1 */
1055 emit_movn_r(ctx, t1, MIPS_R_ZERO, cmp); /* t1 = 0 if cmp != 0 */
1062 emit(ctx, slti, rd, hi(rs), imm < 0 ? -1 : 0); /* rd = rsh < hi(imm) */
1063 emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */
1067 static void emit_slt_r64(struct jit_context *ctx, u8 rd,
1079 emit(ctx, sltu, t1, lo(rs), lo(rt)); /* t1 = rsl <u rtl */
1080 emit(ctx, sltu, t2, lo(rt), lo(rs)); /* t2 = rtl <u rsl */
1081 emit(ctx, xor, t3, hi(rs), hi(rt)); /* t3 = rlh ^ rth */
1082 emit(ctx, srl, rd, t3, 31); /* rd = t3 >> 31 */
1083 emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */
1084 emit_movn_r(ctx, t1, MIPS_R_ZERO, t3); /* t1 = 0 if t3 != 0 */
1087 emit(ctx, slt, rd, hi(rs), hi(rt)); /* rd = rsh <s rth */
1088 emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */
1092 static void emit_jmp_i64(struct jit_context *ctx,
1106 emit(ctx, addiu, tmp, lo(dst), -imm);
1108 emit(ctx, xori, tmp, lo(dst), imm);
1110 emit_mov_i(ctx, tmp, imm);
1111 emit(ctx, xor, tmp, lo(dst), tmp);
1114 emit(ctx, addu, MIPS_R_T9, hi(dst), 1);
1115 emit(ctx, or, tmp, tmp, MIPS_R_T9);
1117 emit(ctx, or, tmp, tmp, hi(dst));
1120 emit(ctx, beqz, tmp, off);
1122 emit(ctx, bnez, tmp, off);
1129 emit(ctx, andi, tmp, lo(dst), imm);
1131 emit_mov_i(ctx, tmp, imm);
1132 emit(ctx, and, tmp, lo(dst), tmp);
1135 emit(ctx, or, tmp, tmp, hi(dst));
1137 emit(ctx, bnez, tmp, off);
1139 emit(ctx, beqz, tmp, off);
1143 emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1);
1144 emit(ctx, beqz, tmp, off);
1148 emit_sltiu_r64(ctx, tmp, dst, imm);
1149 emit(ctx, beqz, tmp, off);
1153 emit_sltiu_r64(ctx, tmp, dst, imm);
1154 emit(ctx, bnez, tmp, off);
1158 emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1);
1159 emit(ctx, bnez, tmp, off);
1163 emit_slti_r64(ctx, tmp, dst, (s64)imm + 1);
1164 emit(ctx, beqz, tmp, off);
1168 emit_slti_r64(ctx, tmp, dst, imm);
1169 emit(ctx, beqz, tmp, off);
1173 emit_slti_r64(ctx, tmp, dst, imm);
1174 emit(ctx, bnez, tmp, off);
1178 emit_slti_r64(ctx, tmp, dst, (s64)imm + 1);
1179 emit(ctx, bnez, tmp, off);
1185 static void emit_jmp_r64(struct jit_context *ctx,
1199 emit(ctx, subu, t1, lo(dst), lo(src));
1200 emit(ctx, subu, t2, hi(dst), hi(src));
1201 emit(ctx, or, t1, t1, t2);
1203 emit(ctx, beqz, t1, off);
1205 emit(ctx, bnez, t1, off);
1211 emit(ctx, and, t1, lo(dst), lo(src));
1212 emit(ctx, and, t2, hi(dst), hi(src));
1213 emit(ctx, or, t1, t1, t2);
1215 emit(ctx, bnez, t1, off);
1217 emit(ctx, beqz, t1, off);
1221 emit_sltu_r64(ctx, t1, src, dst);
1222 emit(ctx, bnez, t1, off);
1226 emit_sltu_r64(ctx, t1, dst, src);
1227 emit(ctx, beqz, t1, off);
1231 emit_sltu_r64(ctx, t1, dst, src);
1232 emit(ctx, bnez, t1, off);
1236 emit_sltu_r64(ctx, t1, src, dst);
1237 emit(ctx, beqz, t1, off);
1241 emit_slt_r64(ctx, t1, src, dst);
1242 emit(ctx, bnez, t1, off);
1246 emit_slt_r64(ctx, t1, dst, src);
1247 emit(ctx, beqz, t1, off);
1251 emit_slt_r64(ctx, t1, dst, src);
1252 emit(ctx, bnez, t1, off);
1256 emit_slt_r64(ctx, t1, src, dst);
1257 emit(ctx, beqz, t1, off);
1263 static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
1269 if (bpf_jit_get_func_addr(ctx->program, insn, false,
1276 push_regs(ctx, JIT_STACK_REGS, 0, JIT_RESERVED_STACK);
1279 emit_mov_i(ctx, MIPS_R_T9, addr);
1280 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
1281 emit(ctx, nop); /* Delay slot */
1283 clobber_reg(ctx, MIPS_R_RA);
1284 clobber_reg(ctx, MIPS_R_V0);
1285 clobber_reg(ctx, MIPS_R_V1);
1290 static int emit_tail_call(struct jit_context *ctx)
1310 emit(ctx, lw, t1, off, ary); /* t1 = ary->map.max_entries*/
1311 emit_load_delay(ctx); /* Load delay slot */
1312 emit(ctx, sltu, t1, ind, t1); /* t1 = ind < t1 */
1313 emit(ctx, beqz, t1, get_offset(ctx, 1)); /* PC += off(1) if t1 == 0 */
1316 emit(ctx, lw, t2, ctx->stack_size, MIPS_R_SP); /* t2 = *(SP + size) */
1317 emit_load_delay(ctx); /* Load delay slot */
1318 emit(ctx, blez, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 <= 0 */
1319 emit(ctx, addiu, t2, t2, -1); /* t2-- (delay slot) */
1320 emit(ctx, sw, t2, ctx->stack_size, MIPS_R_SP); /* *(SP + size) = t2 */
1326 emit(ctx, sll, t1, ind, 2); /* t1 = ind << 2 */
1327 emit(ctx, addu, t1, t1, ary); /* t1 += ary */
1328 emit(ctx, lw, t2, off, t1); /* t2 = *(t1 + off) */
1329 emit_load_delay(ctx); /* Load delay slot */
1332 emit(ctx, beqz, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 == 0 */
1333 emit(ctx, nop); /* Delay slot */
1339 emit(ctx, lw, t1, off, t2); /* t1 = *(t2 + off) */
1340 emit_load_delay(ctx); /* Load delay slot */
1341 emit(ctx, addiu, t1, t1, JIT_TCALL_SKIP); /* t1 += skip (8 or 12) */
1344 build_epilogue(ctx, t1);
1373 void build_prologue(struct jit_context *ctx)
1392 emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
1393 emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP);
1401 emit(ctx, move, lo(r1), MIPS_R_A0);
1407 emit(ctx, move, hi(r1), MIPS_R_ZERO);
1410 if (ctx->accessed & BIT(BPF_REG_FP))
1411 clobber_reg64(ctx, fp);
1414 saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u32);
1418 locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT);
1426 reserved = ctx->stack_used;
1430 emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, -stack);
1433 push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved);
1436 if (ctx->accessed & BIT(BPF_REG_FP))
1437 emit(ctx, addiu, lo(fp), MIPS_R_SP, stack - saved);
1439 ctx->saved_size = saved;
1440 ctx->stack_size = stack;
1444 void build_epilogue(struct jit_context *ctx, int dest_reg)
1447 pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0,
1448 ctx->stack_size - ctx->saved_size);
1454 emit(ctx, move, MIPS_R_V0, MIPS_R_V1);
1458 emit(ctx, jr, dest_reg);
1459 emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size);
1463 int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
1479 emit_mov_i(ctx, lo(dst), imm);
1480 emit_zext_ver(ctx, dst);
1486 emit_mov_i(ctx, hi(dst), 0);
1488 emit_mov_r(ctx, lo(dst), lo(src));
1489 emit_zext_ver(ctx, dst);
1494 emit_alu_i(ctx, lo(dst), 0, BPF_NEG);
1495 emit_zext_ver(ctx, dst);
1520 emit_mov_i(ctx, MIPS_R_T6, imm);
1521 emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code));
1523 emit_alu_i(ctx, lo(dst), val, alu);
1525 emit_zext_ver(ctx, dst);
1549 emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code));
1550 emit_zext_ver(ctx, dst);
1554 emit_mov_se_i64(ctx, dst, imm);
1558 emit_mov_r(ctx, lo(dst), lo(src));
1559 emit_mov_r(ctx, hi(dst), hi(src));
1563 emit_neg_i64(ctx, dst);
1567 emit_alu_i64(ctx, dst, imm, BPF_OP(code));
1578 emit_alu_i64(ctx, dst, imm, BPF_OP(code));
1587 emit_shift_i64(ctx, dst, imm, BPF_OP(code));
1591 emit_mul_i64(ctx, dst, imm);
1601 emit_mov_se_i64(ctx, tmp, imm);
1602 emit_divmod_r64(ctx, dst, tmp, BPF_OP(code));
1614 emit_alu_r64(ctx, dst, src, BPF_OP(code));
1622 emit_shift_r64(ctx, dst, lo(src), BPF_OP(code));
1626 emit_mul_r64(ctx, dst, src);
1632 emit_divmod_r64(ctx, dst, src, BPF_OP(code));
1645 emit_bswap_r64(ctx, dst, imm);
1647 emit_trunc_r64(ctx, dst, imm);
1651 emit_mov_i(ctx, lo(dst), imm);
1652 emit_mov_i(ctx, hi(dst), insn[1].imm);
1659 emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code));
1669 emit_mov_se_i64(ctx, tmp, imm);
1674 emit_mov_i(ctx, lo(tmp), imm);
1677 emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code));
1684 emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code));
1702 emit_atomic_r(ctx, lo(dst), lo(src), off, imm);
1704 emit_atomic_r32(ctx, lo(dst), lo(src),
1707 emit_zext_ver(ctx, src);
1711 emit_cmpxchg_r(ctx, lo(dst), lo(src),
1714 emit_cmpxchg_r32(ctx, lo(dst), lo(src), off);
1733 emit_atomic_r64(ctx, lo(dst), src, off, imm);
1736 emit_cmpxchg_r64(ctx, lo(dst), src, off);
1766 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
1767 emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp);
1768 if (finish_jmp(ctx, jmp, off) < 0)
1795 setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel);
1797 emit_jmp_i(ctx, lo(dst), imm, rel, jmp);
1800 emit_mov_i(ctx, MIPS_R_T6, imm);
1801 emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp);
1803 if (finish_jmp(ctx, jmp, off) < 0)
1830 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
1831 emit_jmp_r64(ctx, dst, src, rel, jmp);
1832 if (finish_jmp(ctx, jmp, off) < 0)
1859 setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel);
1860 emit_jmp_i64(ctx, dst, imm, rel, jmp);
1861 if (finish_jmp(ctx, jmp, off) < 0)
1868 if (emit_ja(ctx, off) < 0)
1873 if (emit_tail_call(ctx) < 0)
1878 if (emit_call(ctx, insn) < 0)
1887 if (ctx->bpf_index == ctx->program->len - 1)
1889 if (emit_exit(ctx) < 0)
1902 ctx->bpf_index, code);