Lines Matching refs:ctx
98 static void emit_sext(struct jit_context *ctx, u8 dst, u8 src)
100 emit(ctx, sll, dst, src, 0);
101 clobber_reg(ctx, dst);
105 static void emit_zext(struct jit_context *ctx, u8 dst)
108 emit(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
110 emit(ctx, and, dst, dst, bpf2mips64[JIT_REG_ZX]);
111 access_reg(ctx, JIT_REG_ZX); /* We need the ZX register */
113 clobber_reg(ctx, dst);
117 static void emit_zext_ver(struct jit_context *ctx, u8 dst)
119 if (!ctx->program->aux->verifier_zext)
120 emit_zext(ctx, dst);
124 static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64)
127 emit(ctx, daddiu, dst, MIPS_R_ZERO, (s16)imm64);
130 emit(ctx, lui, dst, (s16)(imm64 >> 16));
131 emit(ctx, ori, dst, dst, (u16)imm64 & 0xffff);
145 emit(ctx, dsll_safe, dst, dst, shift);
146 emit(ctx, ori, dst, acc, half);
152 emit(ctx, dsll_safe, dst, dst, shift);
154 clobber_reg(ctx, dst);
158 static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op)
163 emit(ctx, ori, dst, dst, (u16)imm);
167 emit(ctx, xori, dst, dst, (u16)imm);
171 emit(ctx, dsubu, dst, MIPS_R_ZERO, dst);
175 emit(ctx, dsll_safe, dst, dst, imm);
179 emit(ctx, dsrl_safe, dst, dst, imm);
183 emit(ctx, dsra_safe, dst, dst, imm);
187 emit(ctx, daddiu, dst, dst, imm);
191 emit(ctx, daddiu, dst, dst, -imm);
195 emit_alu_i(ctx, dst, imm, op);
197 clobber_reg(ctx, dst);
201 static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op)
206 emit(ctx, dsllv, dst, dst, src);
210 emit(ctx, dsrlv, dst, dst, src);
214 emit(ctx, dsrav, dst, dst, src);
218 emit(ctx, daddu, dst, dst, src);
222 emit(ctx, dsubu, dst, dst, src);
227 emit(ctx, dmulu, dst, dst, src);
229 emit(ctx, dmultu, dst, src);
230 emit(ctx, mflo, dst);
233 emit(ctx, mfhi, MIPS_R_ZERO);
239 emit(ctx, ddivu_r6, dst, dst, src);
241 emit(ctx, ddivu, dst, src);
242 emit(ctx, mflo, dst);
248 emit(ctx, dmodu, dst, dst, src);
250 emit(ctx, ddivu, dst, src);
251 emit(ctx, mfhi, dst);
256 emit_alu_r(ctx, dst, src, op);
258 clobber_reg(ctx, dst);
262 static void emit_swap_r64(struct jit_context *ctx, u8 dst, u8 mask, u32 bits)
266 emit(ctx, and, tmp, dst, mask); /* tmp = dst & mask */
267 emit(ctx, dsll, tmp, tmp, bits); /* tmp = tmp << bits */
268 emit(ctx, dsrl, dst, dst, bits); /* dst = dst >> bits */
269 emit(ctx, and, dst, dst, mask); /* dst = dst & mask */
270 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */
274 static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width)
280 emit(ctx, dsbh, dst, dst);
281 emit(ctx, dshd, dst, dst);
286 emit(ctx, dsll32, t2, dst, 0); /* t2 = dst << 32 */
287 emit(ctx, dsrl32, dst, dst, 0); /* dst = dst >> 32 */
288 emit(ctx, or, dst, dst, t2); /* dst = dst | t2 */
290 emit(ctx, ori, t2, MIPS_R_ZERO, 0xffff);
291 emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */
292 emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */
293 emit_swap_r64(ctx, dst, t1, 16);/* dst = swap16(dst) */
295 emit(ctx, lui, t2, 0xff); /* t2 = 0x00ff0000 */
296 emit(ctx, ori, t2, t2, 0xff); /* t2 = t2 | 0x00ff */
297 emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */
298 emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */
299 emit_swap_r64(ctx, dst, t1, 8); /* dst = swap8(dst) */
306 emit_sext(ctx, dst, dst);
307 emit_bswap_r(ctx, dst, width);
309 emit_zext(ctx, dst);
312 clobber_reg(ctx, dst);
316 static void emit_trunc_r64(struct jit_context *ctx, u8 dst, u32 width)
323 emit_zext(ctx, dst);
327 emit(ctx, andi, dst, dst, 0xffff);
330 clobber_reg(ctx, dst);
334 static void emit_ldx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size)
339 emit(ctx, lbu, dst, off, src);
343 emit(ctx, lhu, dst, off, src);
347 emit(ctx, lwu, dst, off, src);
351 emit(ctx, ld, dst, off, src);
354 clobber_reg(ctx, dst);
358 static void emit_stx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size)
363 emit(ctx, sb, src, off, dst);
367 emit(ctx, sh, src, off, dst);
371 emit(ctx, sw, src, off, dst);
375 emit(ctx, sd, src, off, dst);
381 static void emit_atomic_r64(struct jit_context *ctx,
387 LLSC_sync(ctx);
388 emit(ctx, lld, t1, off, dst);
392 emit(ctx, daddu, t2, t1, src);
396 emit(ctx, and, t2, t1, src);
400 emit(ctx, or, t2, t1, src);
404 emit(ctx, xor, t2, t1, src);
407 emit(ctx, move, t2, src);
410 emit(ctx, scd, t2, off, dst);
411 emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset);
412 emit(ctx, nop); /* Delay slot */
415 emit(ctx, move, src, t1);
416 clobber_reg(ctx, src);
421 static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
427 LLSC_sync(ctx);
428 emit(ctx, lld, t1, off, dst);
429 emit(ctx, bne, t1, r0, 12);
430 emit(ctx, move, t2, src); /* Delay slot */
431 emit(ctx, scd, t2, off, dst);
432 emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset);
433 emit(ctx, move, r0, t1); /* Delay slot */
435 clobber_reg(ctx, r0);
439 static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn)
447 if (bpf_jit_get_func_addr(ctx->program, insn, false,
454 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
457 emit_mov_i64(ctx, tmp, addr & JALR_MASK);
458 emit(ctx, jalr, MIPS_R_RA, tmp);
459 emit(ctx, nop); /* Delay slot */
462 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0);
465 if (ctx->accessed & BIT(JIT_REG_ZX)) {
466 emit(ctx, daddiu, zx, MIPS_R_ZERO, -1);
467 emit(ctx, dsrl32, zx, zx, 0);
470 clobber_reg(ctx, MIPS_R_RA);
471 clobber_reg(ctx, MIPS_R_V0);
472 clobber_reg(ctx, MIPS_R_V1);
477 static int emit_tail_call(struct jit_context *ctx)
496 emit(ctx, lwu, tmp, off, ary); /* tmp = ary->map.max_entrs*/
497 emit(ctx, sltu, tmp, ind, tmp); /* tmp = ind < t1 */
498 emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/
501 emit(ctx, daddiu, tcc, tcc, -1); /* tcc-- (delay slot) */
502 emit(ctx, bltz, tcc, get_offset(ctx, 1)); /* PC += off(1) if tcc < 0 */
508 emit(ctx, dsll, tmp, ind, 3); /* tmp = ind << 3 */
509 emit(ctx, daddu, tmp, tmp, ary); /* tmp += ary */
510 emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */
513 emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/
514 emit(ctx, nop); /* Delay slot */
520 emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */
521 emit(ctx, daddiu, tmp, tmp, JIT_TCALL_SKIP); /* tmp += skip (4) */
524 build_epilogue(ctx, tmp);
525 access_reg(ctx, JIT_REG_TC);
546 void build_prologue(struct jit_context *ctx)
566 emit(ctx, ori, tc, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
575 if (ctx->accessed & BIT(BPF_REG_FP))
576 clobber_reg(ctx, fp);
577 if (ctx->accessed & BIT(JIT_REG_TC))
578 clobber_reg(ctx, tc);
579 if (ctx->accessed & BIT(JIT_REG_ZX))
580 clobber_reg(ctx, zx);
583 saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u64);
587 locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT);
594 reserved = ctx->stack_used;
599 emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack);
602 push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved);
605 if (ctx->accessed & BIT(BPF_REG_FP))
606 emit(ctx, daddiu, fp, MIPS_R_SP, stack - saved);
609 if (ctx->accessed & BIT(JIT_REG_ZX)) {
610 emit(ctx, daddiu, zx, MIPS_R_ZERO, -1);
611 emit(ctx, dsrl32, zx, zx, 0);
614 ctx->saved_size = saved;
615 ctx->stack_size = stack;
619 void build_epilogue(struct jit_context *ctx, int dest_reg)
622 pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0,
623 ctx->stack_size - ctx->saved_size);
626 if (ctx->stack_size)
627 emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size);
630 emit(ctx, jr, dest_reg);
631 emit(ctx, sll, MIPS_R_V0, MIPS_R_V0, 0); /* Delay slot */
635 int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
650 emit_mov_i(ctx, dst, imm);
651 emit_zext_ver(ctx, dst);
657 emit_zext(ctx, dst);
659 emit_mov_r(ctx, dst, src);
660 emit_zext_ver(ctx, dst);
665 emit_sext(ctx, dst, dst);
666 emit_alu_i(ctx, dst, 0, BPF_NEG);
667 emit_zext_ver(ctx, dst);
678 emit_mov_i(ctx, MIPS_R_T4, imm);
679 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
681 emit_alu_i(ctx, dst, val, alu);
683 emit_zext_ver(ctx, dst);
700 emit_sext(ctx, dst, dst);
701 emit_mov_i(ctx, MIPS_R_T4, imm);
702 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
704 emit_sext(ctx, dst, dst);
705 emit_alu_i(ctx, dst, val, alu);
707 emit_zext_ver(ctx, dst);
717 emit_alu_r(ctx, dst, src, BPF_OP(code));
718 emit_zext_ver(ctx, dst);
734 emit_sext(ctx, dst, dst);
735 emit_sext(ctx, MIPS_R_T4, src);
736 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
737 emit_zext_ver(ctx, dst);
741 emit_mov_i(ctx, dst, imm);
745 emit_mov_r(ctx, dst, src);
749 emit_alu_i64(ctx, dst, 0, BPF_NEG);
774 emit_mov_i(ctx, MIPS_R_T4, imm);
775 emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code));
777 emit_alu_i64(ctx, dst, val, alu);
802 emit_alu_r64(ctx, dst, src, BPF_OP(code));
815 emit_bswap_r64(ctx, dst, imm);
817 emit_trunc_r64(ctx, dst, imm);
821 emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32));
828 emit_ldx(ctx, dst, src, off, BPF_SIZE(code));
835 emit_mov_i(ctx, MIPS_R_T4, imm);
836 emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code));
843 emit_stx(ctx, dst, src, off, BPF_SIZE(code));
862 emit_atomic_r64(ctx, dst, src, off, imm);
867 emit_mov_r(ctx, MIPS_R_T4, dst);
870 emit_sext(ctx, src, src);
871 emit_atomic_r(ctx, tmp, src, off, imm);
872 emit_zext_ver(ctx, src);
874 emit_sext(ctx, MIPS_R_T4, src);
875 emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm);
880 emit_cmpxchg_r64(ctx, dst, src, off);
886 emit_sext(ctx, tmp, res);
887 emit_sext(ctx, MIPS_R_T5, src);
888 emit_cmpxchg_r(ctx, dst, MIPS_R_T5, tmp, off);
890 emit_mov_r(ctx, res, MIPS_R_T4);
922 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
923 emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */
924 emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */
925 emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp);
926 if (finish_jmp(ctx, jmp, off) < 0)
953 setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel);
954 emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */
956 emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp);
959 emit_mov_i(ctx, MIPS_R_T5, imm);
960 emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp);
962 if (finish_jmp(ctx, jmp, off) < 0)
989 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel);
990 emit_jmp_r(ctx, dst, src, rel, jmp);
991 if (finish_jmp(ctx, jmp, off) < 0)
1018 setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel);
1020 emit_jmp_i(ctx, dst, imm, rel, jmp);
1023 emit_mov_i(ctx, MIPS_R_T4, imm);
1024 emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp);
1026 if (finish_jmp(ctx, jmp, off) < 0)
1033 if (emit_ja(ctx, off) < 0)
1038 if (emit_tail_call(ctx) < 0)
1043 if (emit_call(ctx, insn) < 0)
1052 if (ctx->bpf_index == ctx->program->len - 1)
1054 if (emit_exit(ctx) < 0)
1067 ctx->bpf_index, code);