Lines Matching refs:meta
42 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
44 return meta->l.prev != &nfp_prog->insns;
626 wrp_zext(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst)
628 if (meta->flags & FLAG_INSN_DO_ZEXT)
733 static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
735 bool descending_seq = meta->ldst_gather_len < 0;
736 s16 len = abs(meta->ldst_gather_len);
742 off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
743 src_40bit_addr = meta->ptr.type == PTR_TO_MAP_VALUE;
744 src_base = reg_a(meta->insn.src_reg * 2);
748 addr40_offset(nfp_prog, meta->insn.src_reg * 2, off, &src_base,
765 off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog));
770 reg_a(meta->paired_st->dst_reg * 2), off, len - 1,
775 reg_a(meta->paired_st->dst_reg * 2), off, xfer_num - 1,
782 reg_a(meta->paired_st->dst_reg * 2), off,
789 reg_a(meta->paired_st->dst_reg * 2), off,
796 reg_a(meta->paired_st->dst_reg * 2), off, 7,
799 off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32,
802 reg_a(meta->paired_st->dst_reg * 2), off, len - 33,
813 reg_a(meta->paired_st->dst_reg * 2), off,
815 new_off = meta->paired_st->off + (xfer_num - 1) * 4;
818 xfer_num - 1, reg_a(meta->paired_st->dst_reg * 2), off,
834 else if (BPF_SIZE(meta->insn.code) != BPF_DW)
839 switch (BPF_SIZE(meta->insn.code)) {
841 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2),
846 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2),
850 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2),
854 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2),
856 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1),
861 if (BPF_SIZE(meta->insn.code) != BPF_DW)
862 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
868 data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, swreg offset,
892 wrp_zext(nfp_prog, meta, dst_gpr);
898 data_ld_host_order(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
923 wrp_zext(nfp_prog, meta, dst_gpr);
929 data_ld_host_order_addr32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
932 return data_ld_host_order(nfp_prog, meta, dst_gpr, reg_a(src_gpr),
937 data_ld_host_order_addr40(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
944 return data_ld_host_order(nfp_prog, meta, dst_gpr, rega, regb,
949 construct_data_ind_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
966 return data_ld(nfp_prog, meta, imm_b(nfp_prog), 0, size);
970 construct_data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
982 return data_ld(nfp_prog, meta, tmp_reg, 0, size);
1161 mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1165 s32 off = nfp_prog->stack_frame_depth + meta->insn.off + ptr_off;
1174 if (meta->ptr_not_const ||
1175 meta->flags & FLAG_INSN_PTR_CALLER_STACK_FRAME) {
1180 stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off,
1223 nop_cnt = narrow_ld && meta->flags & FLAG_INSN_DO_ZEXT ? 2 : 3;
1228 wrp_zext(nfp_prog, meta, gpr);
1294 wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1297 const struct bpf_insn *insn = &meta->insn;
1301 meta->flags |= FLAG_INSN_SKIP_NOOP;
1312 wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1315 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
1325 wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1328 const struct bpf_insn *insn = &meta->insn;
1332 wrp_zext(nfp_prog, meta, dst);
1338 wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1341 u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
1344 wrp_zext(nfp_prog, meta, dst);
1358 wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1361 const struct bpf_insn *insn = &meta->insn;
1365 if (is_mbpf_jmp64(meta))
1386 static const struct jmp_code_map *nfp_jmp_code_get(struct nfp_insn_meta *meta)
1390 op = BPF_OP(meta->insn.code) >> 4;
1400 static int cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1402 const struct bpf_insn *insn = &meta->insn;
1409 code = nfp_jmp_code_get(meta);
1413 alu_op = meta->jump_neg_op ? ALU_OP_ADD : ALU_OP_SUB;
1414 carry_op = meta->jump_neg_op ? ALU_OP_ADD_C : ALU_OP_SUB_C;
1422 if (is_mbpf_jmp64(meta)) {
1437 static int cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1439 const struct bpf_insn *insn = &meta->insn;
1443 code = nfp_jmp_code_get(meta);
1457 if (is_mbpf_jmp64(meta))
1501 wrp_mul(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1505 const struct bpf_insn *insn = &meta->insn;
1513 lopnd_max = meta->umax_dst;
1516 ropnd_max = meta->umax_src;
1600 static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1610 if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n))
1680 static int adjust_tail(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1726 map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1734 lm_off += meta->arg2.reg.var_off.value + meta->arg2.reg.off;
1735 load_lm_ptr = meta->arg2.var_off || lm_off;
1740 if (meta->func_id == BPF_FUNC_map_update_elem)
1743 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id,
1767 nfp_get_prandom_u32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1779 nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1784 ptr_type = ur_load_imm_any(nfp_prog, meta->arg1.type, imm_a(nfp_prog));
1788 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id,
1804 nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1811 emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2),
1820 pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2),
1836 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1838 const struct bpf_insn *insn = &meta->insn;
1859 static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1861 u64 imm = meta->insn.imm; /* sign extend */
1863 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U);
1864 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32);
1869 static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1871 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR);
1874 static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1876 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm);
1879 static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1881 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND);
1884 static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1886 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm);
1889 static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1891 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR);
1894 static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1896 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm);
1899 static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1901 const struct bpf_insn *insn = &meta->insn;
1913 static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1915 const struct bpf_insn *insn = &meta->insn;
1924 static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1926 const struct bpf_insn *insn = &meta->insn;
1938 static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1940 const struct bpf_insn *insn = &meta->insn;
1949 static int mul_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1951 return wrp_mul(nfp_prog, meta, true, true);
1954 static int mul_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1956 return wrp_mul(nfp_prog, meta, true, false);
1959 static int div_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1961 const struct bpf_insn *insn = &meta->insn;
1966 static int div_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1971 return wrp_div_imm(nfp_prog, meta->insn.dst_reg * 2, meta->umin_src);
1974 static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1976 const struct bpf_insn *insn = &meta->insn;
2019 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2021 const struct bpf_insn *insn = &meta->insn;
2058 static int shl_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2060 const struct bpf_insn *insn = &meta->insn;
2065 umin = meta->umin_src;
2066 umax = meta->umax_src;
2133 static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2135 const struct bpf_insn *insn = &meta->insn;
2170 static int shr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2172 const struct bpf_insn *insn = &meta->insn;
2177 umin = meta->umin_src;
2178 umax = meta->umax_src;
2245 static int ashr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2247 const struct bpf_insn *insn = &meta->insn;
2287 static int ashr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2289 const struct bpf_insn *insn = &meta->insn;
2294 umin = meta->umin_src;
2295 umax = meta->umax_src;
2326 static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2328 const struct bpf_insn *insn = &meta->insn;
2336 static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2338 const struct bpf_insn *insn = &meta->insn;
2346 static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2348 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR);
2351 static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2353 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR);
2356 static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2358 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND);
2361 static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2363 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND);
2366 static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2368 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR);
2371 static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2373 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR);
2376 static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2378 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD);
2381 static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2383 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD);
2386 static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2388 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB);
2391 static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2393 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB);
2396 static int mul_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2398 return wrp_mul(nfp_prog, meta, false, true);
2401 static int mul_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2403 return wrp_mul(nfp_prog, meta, false, false);
2406 static int div_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2408 return div_reg64(nfp_prog, meta);
2411 static int div_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2413 return div_imm64(nfp_prog, meta);
2416 static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2418 u8 dst = meta->insn.dst_reg * 2;
2421 wrp_zext(nfp_prog, meta, dst);
2427 __ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
2437 wrp_zext(nfp_prog, meta, dst);
2442 static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2444 const struct bpf_insn *insn = &meta->insn;
2449 umin = meta->umin_src;
2450 umax = meta->umax_src;
2452 return __ashr_imm(nfp_prog, meta, dst, umin);
2461 wrp_zext(nfp_prog, meta, dst);
2466 static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2468 const struct bpf_insn *insn = &meta->insn;
2471 return __ashr_imm(nfp_prog, meta, dst, insn->imm);
2475 __shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
2481 wrp_zext(nfp_prog, meta, dst);
2485 static int shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2487 const struct bpf_insn *insn = &meta->insn;
2490 return __shr_imm(nfp_prog, meta, dst, insn->imm);
2493 static int shr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2495 const struct bpf_insn *insn = &meta->insn;
2500 umin = meta->umin_src;
2501 umax = meta->umax_src;
2503 return __shr_imm(nfp_prog, meta, dst, umin);
2509 wrp_zext(nfp_prog, meta, dst);
2514 __shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
2520 wrp_zext(nfp_prog, meta, dst);
2524 static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2526 const struct bpf_insn *insn = &meta->insn;
2529 return __shl_imm(nfp_prog, meta, dst, insn->imm);
2532 static int shl_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2534 const struct bpf_insn *insn = &meta->insn;
2539 umin = meta->umin_src;
2540 umax = meta->umax_src;
2542 return __shl_imm(nfp_prog, meta, dst, umin);
2546 wrp_zext(nfp_prog, meta, dst);
2550 static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2552 const struct bpf_insn *insn = &meta->insn;
2579 static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2581 struct nfp_insn_meta *prev = nfp_meta_prev(meta);
2587 imm_hi = meta->insn.imm;
2600 static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2602 meta->double_cb = imm_ld8_part2;
2606 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2608 return construct_data_ld(nfp_prog, meta, meta->insn.imm, 1);
2611 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2613 return construct_data_ld(nfp_prog, meta, meta->insn.imm, 2);
2616 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2618 return construct_data_ld(nfp_prog, meta, meta->insn.imm, 4);
2621 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2623 return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
2624 meta->insn.src_reg * 2, 1);
2627 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2629 return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
2630 meta->insn.src_reg * 2, 2);
2633 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2635 return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
2636 meta->insn.src_reg * 2, 4);
2640 mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2643 return mem_op_stack(nfp_prog, meta, size, ptr_off,
2644 meta->insn.dst_reg * 2, meta->insn.src_reg * 2,
2648 static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2651 swreg dst = reg_both(meta->insn.dst_reg * 2);
2653 switch (meta->insn.off) {
2674 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
2679 static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2682 swreg dst = reg_both(meta->insn.dst_reg * 2);
2684 switch (meta->insn.off) {
2700 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
2706 mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2711 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
2713 return data_ld_host_order_addr32(nfp_prog, meta, meta->insn.src_reg * 2,
2714 tmp_reg, meta->insn.dst_reg * 2, size);
2718 mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2723 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
2725 return data_ld_host_order_addr40(nfp_prog, meta, meta->insn.src_reg * 2,
2726 tmp_reg, meta->insn.dst_reg * 2, size);
2731 struct nfp_insn_meta *meta)
2733 s16 range_start = meta->pkt_cache.range_start;
2734 s16 range_end = meta->pkt_cache.range_end;
2740 src_base = reg_a(meta->insn.src_reg * 2);
2757 struct nfp_insn_meta *meta,
2760 s16 range_start = meta->pkt_cache.range_start;
2761 s16 insn_off = meta->insn.off - range_start;
2763 u8 dst_gpr = meta->insn.dst_reg * 2;
2786 wrp_zext(nfp_prog, meta, dst_gpr);
2794 wrp_zext(nfp_prog, meta, dst_gpr);
2811 struct nfp_insn_meta *meta,
2817 idx = (meta->insn.off - meta->pkt_cache.range_start) / REG_WIDTH;
2818 dst_gpr = meta->insn.dst_reg * 2;
2825 wrp_zext(nfp_prog, meta, dst_gpr);
2828 wrp_zext(nfp_prog, meta, dst_gpr);
2841 struct nfp_insn_meta *meta, unsigned int size)
2843 u8 off = meta->insn.off - meta->pkt_cache.range_start;
2846 return mem_ldx_data_from_pktcache_aligned(nfp_prog, meta, size);
2848 return mem_ldx_data_from_pktcache_unaligned(nfp_prog, meta, size);
2852 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2855 if (meta->ldst_gather_len)
2856 return nfp_cpp_memcpy(nfp_prog, meta);
2858 if (meta->ptr.type == PTR_TO_CTX) {
2860 return mem_ldx_xdp(nfp_prog, meta, size);
2862 return mem_ldx_skb(nfp_prog, meta, size);
2865 if (meta->ptr.type == PTR_TO_PACKET) {
2866 if (meta->pkt_cache.range_end) {
2867 if (meta->pkt_cache.do_init)
2868 mem_ldx_data_init_pktcache(nfp_prog, meta);
2870 return mem_ldx_data_from_pktcache(nfp_prog, meta, size);
2872 return mem_ldx_data(nfp_prog, meta, size);
2876 if (meta->ptr.type == PTR_TO_STACK)
2877 return mem_ldx_stack(nfp_prog, meta, size,
2878 meta->ptr.off + meta->ptr.var_off.value);
2880 if (meta->ptr.type == PTR_TO_MAP_VALUE)
2881 return mem_ldx_emem(nfp_prog, meta, size);
2886 static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2888 return mem_ldx(nfp_prog, meta, 1);
2891 static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2893 return mem_ldx(nfp_prog, meta, 2);
2896 static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2898 return mem_ldx(nfp_prog, meta, 4);
2901 static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2903 return mem_ldx(nfp_prog, meta, 8);
2907 mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2910 u64 imm = meta->insn.imm; /* sign extend */
2913 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
2915 return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg,
2919 static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2922 if (meta->ptr.type == PTR_TO_PACKET)
2923 return mem_st_data(nfp_prog, meta, size);
2928 static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2930 return mem_st(nfp_prog, meta, 1);
2933 static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2935 return mem_st(nfp_prog, meta, 2);
2938 static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2940 return mem_st(nfp_prog, meta, 4);
2943 static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2945 return mem_st(nfp_prog, meta, 8);
2949 mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2954 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
2956 return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg,
2957 meta->insn.src_reg * 2, size);
2961 mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2964 return mem_op_stack(nfp_prog, meta, size, ptr_off,
2965 meta->insn.src_reg * 2, meta->insn.dst_reg * 2,
2969 static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2971 switch (meta->insn.off) {
2973 return nfp_queue_select(nfp_prog, meta);
2981 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2984 if (meta->ptr.type == PTR_TO_PACKET)
2985 return mem_stx_data(nfp_prog, meta, size);
2987 if (meta->ptr.type == PTR_TO_STACK)
2988 return mem_stx_stack(nfp_prog, meta, size,
2989 meta->ptr.off + meta->ptr.var_off.value);
2994 static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2996 return mem_stx(nfp_prog, meta, 1);
2999 static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3001 return mem_stx(nfp_prog, meta, 2);
3004 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3006 if (meta->ptr.type == PTR_TO_CTX)
3008 return mem_stx_xdp(nfp_prog, meta);
3009 return mem_stx(nfp_prog, meta, 4);
3012 static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3014 return mem_stx(nfp_prog, meta, 8);
3018 mem_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, bool is64)
3020 u8 dst_gpr = meta->insn.dst_reg * 2;
3021 u8 src_gpr = meta->insn.src_reg * 2;
3025 off = ur_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
3035 if (meta->insn.off) {
3039 if (meta->xadd_maybe_16bit) {
3043 if (meta->xadd_over_16bit)
3045 if (meta->xadd_maybe_16bit && meta->xadd_over_16bit) {
3051 if (meta->xadd_maybe_16bit && meta->xadd_over_16bit) {
3059 emit_br(nfp_prog, BR_BLO, full_add, meta->insn.off ? 2 : 0);
3064 if (!meta->insn.off) {
3077 if (meta->xadd_maybe_16bit) {
3088 if (meta->xadd_over_16bit)
3096 if (meta->xadd_over_16bit) {
3112 static int mem_atomic4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3114 if (meta->insn.imm != BPF_ADD)
3117 return mem_xadd(nfp_prog, meta, false);
3120 static int mem_atomic8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3122 if (meta->insn.imm != BPF_ADD)
3125 return mem_xadd(nfp_prog, meta, true);
3128 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3130 emit_br(nfp_prog, BR_UNC, meta->insn.off, 0);
3135 static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3137 const struct bpf_insn *insn = &meta->insn;
3164 static int jeq32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3166 const struct bpf_insn *insn = &meta->insn;
3177 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3179 const struct bpf_insn *insn = &meta->insn;
3190 if (is_mbpf_jmp64(meta) && imm >> 32) {
3199 static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3201 const struct bpf_insn *insn = &meta->insn;
3203 bool is_jmp32 = is_mbpf_jmp32(meta);
3233 static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3235 const struct bpf_insn *insn = &meta->insn;
3239 if (is_mbpf_jmp64(meta)) {
3251 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3253 return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
3256 static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3258 return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE);
3262 bpf_to_bpf_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3311 if (!meta->jmp_dst) {
3315 if (nfp_prog->subprog[meta->jmp_dst->subprog_idx].needs_reg_push) {
3323 emit_br(nfp_prog, BR_UNC, meta->insn.imm, 1);
3341 meta->num_insns_after_br = nfp_prog_current_offset(nfp_prog);
3342 meta->num_insns_after_br -= offset_br;
3347 static int helper_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3349 switch (meta->insn.imm) {
3351 return adjust_head(nfp_prog, meta);
3353 return adjust_tail(nfp_prog, meta);
3357 return map_call_stack_common(nfp_prog, meta);
3359 return nfp_get_prandom_u32(nfp_prog, meta);
3361 return nfp_perf_event_output(nfp_prog, meta);
3368 static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3370 if (is_mbpf_pseudo_call(meta))
3371 return bpf_to_bpf_call(nfp_prog, meta);
3373 return helper_call(nfp_prog, meta);
3376 static bool nfp_is_main_function(struct nfp_insn_meta *meta)
3378 return meta->subprog_idx == 0;
3381 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3389 nfp_subprog_epilogue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3391 if (nfp_prog->subprog[meta->subprog_idx].needs_reg_push) {
3413 static int jmp_exit(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3415 if (nfp_is_main_function(meta))
3416 return goto_out(nfp_prog, meta);
3418 return nfp_subprog_epilogue(nfp_prog, meta);
3541 nfp_fixup_immed_relo(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
3556 struct nfp_insn_meta *meta, *jmp_dst;
3560 list_for_each_entry(meta, &nfp_prog->insns, l) {
3561 if (meta->flags & FLAG_INSN_SKIP_MASK)
3563 if (!is_mbpf_jmp(meta))
3565 if (meta->insn.code == (BPF_JMP | BPF_EXIT) &&
3566 !nfp_is_main_function(meta))
3568 if (is_mbpf_helper_call(meta))
3571 if (list_is_last(&meta->l, &nfp_prog->insns))
3574 br_idx = list_next_entry(meta, l)->off - 1;
3581 if (is_mbpf_pseudo_call(meta))
3582 br_idx -= meta->num_insns_after_br;
3586 br_idx, meta->insn.code, nfp_prog->prog[br_idx]);
3590 if (meta->insn.code == (BPF_JMP | BPF_EXIT))
3595 RELO_BR_REL && !is_mbpf_pseudo_call(meta))
3598 if (!meta->jmp_dst) {
3603 jmp_dst = meta->jmp_dst;
3610 if (is_mbpf_pseudo_call(meta) &&
3612 err = nfp_fixup_immed_relo(nfp_prog, meta,
3622 for (idx = meta->off; idx <= br_idx; idx++) {
3640 nfp_subprog_prologue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3647 nfp_start_subprog(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3649 unsigned int depth = nfp_prog->subprog[meta->subprog_idx].stack_depth;
3652 nfp_subprog_prologue(nfp_prog, meta);
3655 bool nfp_is_subprog_start(struct nfp_insn_meta *meta)
3657 return meta->flags & FLAG_INSN_IS_SUBPROG_START;
3833 struct nfp_insn_meta *meta;
3844 list_for_each_entry(meta, &nfp_prog->insns, l) {
3845 instr_cb_t cb = instr_cb[meta->insn.code];
3847 meta->off = nfp_prog_current_offset(nfp_prog);
3849 if (nfp_is_subprog_start(meta)) {
3850 nfp_start_subprog(nfp_prog, meta);
3855 if (meta->flags & FLAG_INSN_SKIP_MASK) {
3860 if (nfp_meta_has_prev(nfp_prog, meta) &&
3861 nfp_meta_prev(meta)->double_cb)
3862 cb = nfp_meta_prev(meta)->double_cb;
3865 err = cb(nfp_prog, meta);
3890 struct nfp_insn_meta *meta;
3892 list_for_each_entry(meta, &nfp_prog->insns, l) {
3893 struct bpf_insn insn = meta->insn;
3903 meta->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;
3906 if (!(meta->flags & FLAG_INSN_SKIP_MASK))
3916 struct nfp_insn_meta *meta;
3918 list_for_each_entry(meta, &nfp_prog->insns, l) {
3919 struct bpf_insn insn = meta->insn;
3921 if (meta->flags & FLAG_INSN_SKIP_MASK)
3924 if (!is_mbpf_alu(meta) && !is_mbpf_jmp(meta))
3931 if (is_mbpf_jmp(meta)) {
3937 meta->jump_neg_op = true;
3950 meta->insn.code = insn.code | BPF_K;
3953 meta->insn.imm = -insn.imm;
4278 struct nfp_insn_meta *meta, *range_node = NULL;
4285 list_for_each_entry(meta, &nfp_prog->insns, l) {
4286 if (meta->flags & FLAG_INSN_IS_JUMP_DST)
4289 if (meta->flags & FLAG_INSN_SKIP_MASK)
4292 insn = &meta->insn;
4294 if (is_mbpf_store_pkt(meta) ||
4296 is_mbpf_classic_store_pkt(meta) ||
4297 is_mbpf_classic_load(meta)) {
4302 if (!is_mbpf_load(meta))
4305 if (meta->ptr.type != PTR_TO_PACKET || meta->ldst_gather_len) {
4328 if (meta->ptr.id == range_ptr_id &&
4329 meta->ptr.off == range_ptr_off) {
4361 range_node = meta;
4374 list_for_each_entry(meta, &nfp_prog->insns, l) {
4375 if (meta->flags & FLAG_INSN_SKIP_MASK)
4378 if (is_mbpf_load_pkt(meta) && !meta->ldst_gather_len) {
4379 if (meta->pkt_cache.do_init) {
4380 range_start = meta->pkt_cache.range_start;
4381 range_end = meta->pkt_cache.range_end;
4383 meta->pkt_cache.range_start = range_start;
4384 meta->pkt_cache.range_end = range_end;
4493 struct nfp_insn_meta *meta;
4496 list_for_each_entry(meta, &nfp_prog->insns, l) {
4498 u64 code = meta->insn.code;
4502 if (!is_mbpf_jmp(meta))
4506 if (is_mbpf_helper_call(meta))
4515 dst_idx = meta->n + 1 + meta->insn.imm;
4517 dst_idx = meta->n + 1 + meta->insn.off;
4519 dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_idx);
4525 meta->jmp_dst = dst_meta;