Lines Matching refs:gen
45 static int blob_fd_array_off(struct bpf_gen *gen, int index)
47 return gen->fd_array + index * sizeof(int);
50 static int realloc_insn_buf(struct bpf_gen *gen, __u32 size)
52 size_t off = gen->insn_cur - gen->insn_start;
55 if (gen->error)
56 return gen->error;
58 gen->error = -ERANGE;
61 insn_start = realloc(gen->insn_start, off + size);
63 gen->error = -ENOMEM;
64 free(gen->insn_start);
65 gen->insn_start = NULL;
68 gen->insn_start = insn_start;
69 gen->insn_cur = insn_start + off;
73 static int realloc_data_buf(struct bpf_gen *gen, __u32 size)
75 size_t off = gen->data_cur - gen->data_start;
78 if (gen->error)
79 return gen->error;
81 gen->error = -ERANGE;
84 data_start = realloc(gen->data_start, off + size);
86 gen->error = -ENOMEM;
87 free(gen->data_start);
88 gen->data_start = NULL;
91 gen->data_start = data_start;
92 gen->data_cur = data_start + off;
96 static void emit(struct bpf_gen *gen, struct bpf_insn insn)
98 if (realloc_insn_buf(gen, sizeof(insn)))
100 memcpy(gen->insn_cur, &insn, sizeof(insn));
101 gen->insn_cur += sizeof(insn);
104 static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2)
106 emit(gen, insn1);
107 emit(gen, insn2);
110 static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
111 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
113 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps)
118 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
119 gen->log_level = log_level;
121 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1));
124 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
125 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz));
126 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz));
127 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
128 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
133 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0,
139 nr_maps * (6 + (gen->log_level ? 6 : 0))));
142 gen->cleanup_label = gen->insn_cur - gen->insn_start;
145 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i));
146 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1));
147 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
150 emit_sys_close_blob(gen, blob_fd_array_off(gen, i));
152 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
153 emit(gen, BPF_EXIT_INSN());
156 static int add_data(struct bpf_gen *gen, const void *data, __u32 size)
162 if (realloc_data_buf(gen, size8))
164 prev = gen->data_cur;
166 memcpy(gen->data_cur, data, size);
167 memcpy(gen->data_cur + size, &zero, size8 - size);
169 memset(gen->data_cur, 0, size8);
171 gen->data_cur += size8;
172 return prev - gen->data_start;
178 static int add_map_fd(struct bpf_gen *gen)
180 if (gen->nr_maps == MAX_USED_MAPS) {
182 gen->error = -E2BIG;
185 return gen->nr_maps++;
188 static int add_kfunc_btf_fd(struct bpf_gen *gen)
192 if (gen->nr_fd_array == MAX_KFUNC_DESCS) {
193 cur = add_data(gen, NULL, sizeof(int));
194 return (cur - gen->fd_array) / sizeof(int);
196 return MAX_USED_MAPS + gen->nr_fd_array++;
211 static void emit_rel_store(struct bpf_gen *gen, int off, int data)
213 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
215 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
217 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
220 static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off)
222 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
224 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0));
225 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
227 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
230 static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off)
232 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
234 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0));
235 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
238 static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,
241 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off));
246 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3));
247 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
249 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
252 static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off)
254 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
255 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
257 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
260 static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off)
262 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
263 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
266 static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size)
268 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd));
269 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
271 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size));
272 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf));
274 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
282 static void emit_check_err(struct bpf_gen *gen)
284 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1;
290 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off));
292 gen->error = -ERANGE;
293 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1));
298 static void emit_debug(struct bpf_gen *gen, int reg1, int reg2,
304 if (!gen->log_level)
314 addr = add_data(gen, buf, len);
316 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
318 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
320 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1));
322 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2));
323 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk));
326 static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...)
331 emit_debug(gen, reg1, reg2, fmt, args);
335 static void debug_ret(struct bpf_gen *gen, const char *fmt, ...)
340 emit_debug(gen, BPF_REG_7, -1, fmt, args);
344 static void __emit_sys_close(struct bpf_gen *gen)
346 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0,
350 2 + (gen->log_level ? 6 : 0)));
351 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1));
352 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
353 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d");
356 static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off)
358 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off));
359 __emit_sys_close(gen);
362 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off)
364 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
366 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0));
367 __emit_sys_close(gen);
370 int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps)
374 if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) {
376 nr_progs, gen->nr_progs, nr_maps, gen->nr_maps);
377 gen->error = -EFAULT;
378 return gen->error;
380 emit_sys_close_stack(gen, stack_off(btf_fd));
381 for (i = 0; i < gen->nr_progs; i++)
382 move_stack2ctx(gen,
384 sizeof(struct bpf_map_desc) * gen->nr_maps +
388 for (i = 0; i < gen->nr_maps; i++)
389 move_blob2ctx(gen,
393 blob_fd_array_off(gen, i));
394 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
395 emit(gen, BPF_EXIT_INSN());
396 pr_debug("gen: finish %d\n", gen->error);
397 if (!gen->error) {
398 struct gen_loader_opts *opts = gen->opts;
400 opts->insns = gen->insn_start;
401 opts->insns_sz = gen->insn_cur - gen->insn_start;
402 opts->data = gen->data_start;
403 opts->data_sz = gen->data_cur - gen->data_start;
405 return gen->error;
408 void bpf_gen__free(struct bpf_gen *gen)
410 if (!gen)
412 free(gen->data_start);
413 free(gen->insn_start);
414 free(gen);
417 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
425 pr_debug("gen: load_btf: size %d\n", btf_raw_size);
426 btf_data = add_data(gen, btf_raw_data, btf_raw_size);
429 btf_load_attr = add_data(gen, &attr, attr_size);
432 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,
434 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4,
436 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8,
439 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data);
441 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size);
442 debug_ret(gen, "btf_load size %d", btf_raw_size);
443 emit_check_err(gen);
445 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd)));
448 void bpf_gen__map_create(struct bpf_gen *gen,
473 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d\n",
476 map_create_attr = add_data(gen, &attr, attr_size);
479 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,
484 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,
493 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4,
499 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);
500 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",
503 emit_check_err(gen);
510 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
512 } else if (map_idx != gen->nr_maps) {
513 gen->error = -EDOM; /* internal bug */
516 /* add_map_fd does gen->nr_maps++ */
517 idx = add_map_fd(gen);
518 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
519 0, 0, 0, blob_fd_array_off(gen, idx)));
520 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0));
523 emit_sys_close_stack(gen, stack_off(inner_map_fd));
526 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name,
533 gen->attach_kind = kind;
534 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s",
536 if (ret >= sizeof(gen->attach_target))
537 gen->error = -ENOSPC;
540 static void emit_find_attach_target(struct bpf_gen *gen)
542 int name, len = strlen(gen->attach_target) + 1;
544 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind);
545 name = add_data(gen, gen->attach_target, len);
547 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
549 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
550 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind));
551 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
552 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
553 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
554 debug_ret(gen, "find_by_name_kind(%s,%d)",
555 gen->attach_target, gen->attach_kind);
556 emit_check_err(gen);
562 void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak,
567 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo));
569 gen->error = -ENOMEM;
572 gen->relos = relo;
573 relo += gen->relo_cnt;
580 gen->relo_cnt++;
584 static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo)
589 for (i = 0; i < gen->nr_ksyms; i++) {
590 kdesc = &gen->ksyms[i];
597 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc));
599 gen->error = -ENOMEM;
602 gen->ksyms = kdesc;
603 kdesc = &gen->ksyms[gen->nr_ksyms++];
616 static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo)
620 name_off = add_data(gen, relo->name, len);
621 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
623 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
624 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));
625 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
626 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
627 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
628 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);
635 static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo)
639 name_off = add_data(gen, relo->name, len);
640 res_off = add_data(gen, NULL, 8); /* res is u64 */
641 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
643 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
644 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
645 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE,
647 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4));
648 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name));
649 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0));
650 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
651 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind);
664 static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
669 kdesc = get_ksym_desc(gen, relo);
674 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
676 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2,
682 emit_bpf_find_by_name_kind(gen, relo);
684 emit_check_err(gen);
686 btf_fd_idx = add_kfunc_btf_fd(gen);
690 gen->error = -E2BIG;
695 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
697 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
698 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
700 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10));
702 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
704 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7));
705 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
707 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
708 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx)));
710 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0));
712 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2));
714 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
716 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
718 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx));
720 if (!gen->log_level)
722 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
724 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8,
726 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d",
728 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
729 0, 0, 0, blob_fd_array_off(gen, kdesc->off)));
730 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0));
731 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd",
735 static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo,
738 if (!gen->log_level)
740 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
742 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) +
744 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d",
746 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
747 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg",
754 static void emit_relo_ksym_typeless(struct bpf_gen *gen,
759 kdesc = get_ksym_desc(gen, relo);
764 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
766 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
774 emit_bpf_kallsyms_lookup_name(gen, relo);
775 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1));
776 emit_check_err(gen);
778 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm)));
780 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
781 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9,
784 emit_ksym_relo_log(gen, relo, kdesc->ref);
801 static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
806 kdesc = get_ksym_desc(gen, relo);
811 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
813 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
818 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3));
823 emit_bpf_find_by_name_kind(gen, relo);
825 emit_check_err(gen);
827 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
829 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
830 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0));
832 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4));
834 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
836 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
837 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7,
840 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3));
844 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
845 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask));
846 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code)));
848 emit_ksym_relo_log(gen, relo, kdesc->ref);
851 void bpf_gen__record_relo_core(struct bpf_gen *gen,
856 relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos));
858 gen->error = -ENOMEM;
861 gen->core_relos = relos;
862 relos += gen->core_relo_cnt;
864 gen->core_relo_cnt++;
867 static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
871 pr_debug("gen: emit_relo (%d): %s at %d %s\n",
874 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn));
877 emit_relo_ksym_typeless(gen, relo, insn);
879 emit_relo_ksym_btf(gen, relo, insn);
881 emit_relo_kfunc_btf(gen, relo, insn);
885 static void emit_relos(struct bpf_gen *gen, int insns)
889 for (i = 0; i < gen->relo_cnt; i++)
890 emit_relo(gen, gen->relos + i, insns);
893 static void cleanup_core_relo(struct bpf_gen *gen)
895 if (!gen->core_relo_cnt)
897 free(gen->core_relos);
898 gen->core_relo_cnt = 0;
899 gen->core_relos = NULL;
902 static void cleanup_relos(struct bpf_gen *gen, int insns)
907 for (i = 0; i < gen->nr_ksyms; i++) {
908 kdesc = &gen->ksyms[i];
914 emit_sys_close_blob(gen, insn);
916 emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off));
918 gen->nr_fd_array--;
921 if (gen->nr_ksyms) {
922 free(gen->ksyms);
923 gen->nr_ksyms = 0;
924 gen->ksyms = NULL;
926 if (gen->relo_cnt) {
927 free(gen->relos);
928 gen->relo_cnt = 0;
929 gen->relos = NULL;
931 cleanup_core_relo(gen);
934 void bpf_gen__prog_load(struct bpf_gen *gen,
944 pr_debug("gen: prog_load: type %d insns_cnt %zd progi_idx %d\n",
947 license_off = add_data(gen, license, strlen(license) + 1);
949 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));
961 func_info = add_data(gen, load_attr->func_info,
966 line_info = add_data(gen, load_attr->line_info,
970 attr.core_relo_cnt = gen->core_relo_cnt;
971 core_relos = add_data(gen, gen->core_relos,
975 prog_load_attr = add_data(gen, &attr, attr_size);
978 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off);
981 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off);
984 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info);
987 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
990 emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos);
993 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);
996 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,
998 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4,
1000 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8,
1003 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4,
1005 if (gen->attach_kind) {
1006 emit_find_attach_target(gen);
1008 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
1010 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
1012 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
1013 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
1016 emit_relos(gen, insns_off);
1018 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size);
1019 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt);
1021 cleanup_relos(gen, insns_off);
1022 if (gen->attach_kind) {
1023 emit_sys_close_blob(gen,
1025 gen->attach_kind = 0;
1027 emit_check_err(gen);
1029 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
1030 stack_off(prog_fd[gen->nr_progs])));
1031 gen->nr_progs++;
1034 void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
1043 pr_debug("gen: map_update_elem: idx %d\n", map_idx);
1045 value = add_data(gen, pvalue, value_size);
1046 key = add_data(gen, &zero, sizeof(zero));
1055 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6,
1059 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8));
1060 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
1062 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size));
1063 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6,
1065 emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2));
1066 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
1067 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
1068 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
1070 map_update_attr = add_data(gen, &attr, attr_size);
1071 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1072 blob_fd_array_off(gen, map_idx));
1073 emit_rel_store(gen, attr_field(map_update_attr, key), key);
1074 emit_rel_store(gen, attr_field(map_update_attr, value), value);
1076 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
1077 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size);
1078 emit_check_err(gen);
1081 void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot,
1089 pr_debug("gen: populate_outer_map: outer %d key %d inner %d\n",
1092 key = add_data(gen, &slot, sizeof(slot));
1094 map_update_attr = add_data(gen, &attr, attr_size);
1095 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1096 blob_fd_array_off(gen, outer_map_idx));
1097 emit_rel_store(gen, attr_field(map_update_attr, key), key);
1098 emit_rel_store(gen, attr_field(map_update_attr, value),
1099 blob_fd_array_off(gen, inner_map_idx));
1102 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
1103 debug_ret(gen, "populate_outer_map outer %d key %d inner %d",
1105 emit_check_err(gen);
1108 void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx)
1115 pr_debug("gen: map_freeze: idx %d\n", map_idx);
1116 map_freeze_attr = add_data(gen, &attr, attr_size);
1117 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
1118 blob_fd_array_off(gen, map_idx));
1120 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);
1121 debug_ret(gen, "map_freeze");
1122 emit_check_err(gen);