1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Common code for probe-based Dynamic events.
4 *
5 * This code was copied from kernel/trace/trace_kprobe.c written by
6 * Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
7 *
8 * Updates to make this generic:
9 * Copyright (C) IBM Corporation, 2010-2011
10 * Author:     Srikar Dronamraju
11 */
12#define pr_fmt(fmt)	"trace_probe: " fmt
13
14#include <linux/bpf.h>
15#include "trace_btf.h"
16
17#include "trace_probe.h"
18
19#undef C
20#define C(a, b)		b
21
22static const char *trace_probe_err_text[] = { ERRORS };
23
24static const char *reserved_field_names[] = {
25	"common_type",
26	"common_flags",
27	"common_preempt_count",
28	"common_pid",
29	"common_tgid",
30	FIELD_STRING_IP,
31	FIELD_STRING_RETIP,
32	FIELD_STRING_FUNC,
33};
34
35/* Printing  in basic type function template */
36#define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt)			\
37int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
38{									\
39	trace_seq_printf(s, fmt, *(type *)data);			\
40	return !trace_seq_has_overflowed(s);				\
41}									\
42const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
43
44DEFINE_BASIC_PRINT_TYPE_FUNC(u8,  u8,  "%u")
45DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
46DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
47DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
48DEFINE_BASIC_PRINT_TYPE_FUNC(s8,  s8,  "%d")
49DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
50DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
51DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
52DEFINE_BASIC_PRINT_TYPE_FUNC(x8,  u8,  "0x%x")
53DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
54DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
55DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
56DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
57
58int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
59{
60	trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
61	return !trace_seq_has_overflowed(s);
62}
63const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
64
65/* Print type function for string type */
66int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
67{
68	int len = *(u32 *)data >> 16;
69
70	if (!len)
71		trace_seq_puts(s, FAULT_STRING);
72	else
73		trace_seq_printf(s, "\"%s\"",
74				 (const char *)get_loc_data(data, ent));
75	return !trace_seq_has_overflowed(s);
76}
77
78const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
79
80/* Fetch type information table */
81static const struct fetch_type probe_fetch_types[] = {
82	/* Special types */
83	__ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
84			    "__data_loc char[]"),
85	__ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
86			    "__data_loc char[]"),
87	__ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
88			    "__data_loc char[]"),
89	/* Basic types */
90	ASSIGN_FETCH_TYPE(u8,  u8,  0),
91	ASSIGN_FETCH_TYPE(u16, u16, 0),
92	ASSIGN_FETCH_TYPE(u32, u32, 0),
93	ASSIGN_FETCH_TYPE(u64, u64, 0),
94	ASSIGN_FETCH_TYPE(s8,  u8,  1),
95	ASSIGN_FETCH_TYPE(s16, u16, 1),
96	ASSIGN_FETCH_TYPE(s32, u32, 1),
97	ASSIGN_FETCH_TYPE(s64, u64, 1),
98	ASSIGN_FETCH_TYPE_ALIAS(x8,  u8,  u8,  0),
99	ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
100	ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
101	ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
102	ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8,  0),
103	ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
104
105	ASSIGN_FETCH_TYPE_END
106};
107
108static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
109{
110	int i;
111
112	/* Reject the symbol/symstr for uprobes */
113	if (type && (flags & TPARG_FL_USER) &&
114	    (!strcmp(type, "symbol") || !strcmp(type, "symstr")))
115		return NULL;
116
117	if (!type)
118		type = DEFAULT_FETCH_TYPE_STR;
119
120	/* Special case: bitfield */
121	if (*type == 'b') {
122		unsigned long bs;
123
124		type = strchr(type, '/');
125		if (!type)
126			goto fail;
127
128		type++;
129		if (kstrtoul(type, 0, &bs))
130			goto fail;
131
132		switch (bs) {
133		case 8:
134			return find_fetch_type("u8", flags);
135		case 16:
136			return find_fetch_type("u16", flags);
137		case 32:
138			return find_fetch_type("u32", flags);
139		case 64:
140			return find_fetch_type("u64", flags);
141		default:
142			goto fail;
143		}
144	}
145
146	for (i = 0; probe_fetch_types[i].name; i++) {
147		if (strcmp(type, probe_fetch_types[i].name) == 0)
148			return &probe_fetch_types[i];
149	}
150
151fail:
152	return NULL;
153}
154
155static struct trace_probe_log trace_probe_log;
156
157void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
158{
159	trace_probe_log.subsystem = subsystem;
160	trace_probe_log.argc = argc;
161	trace_probe_log.argv = argv;
162	trace_probe_log.index = 0;
163}
164
165void trace_probe_log_clear(void)
166{
167	memset(&trace_probe_log, 0, sizeof(trace_probe_log));
168}
169
170void trace_probe_log_set_index(int index)
171{
172	trace_probe_log.index = index;
173}
174
175void __trace_probe_log_err(int offset, int err_type)
176{
177	char *command, *p;
178	int i, len = 0, pos = 0;
179
180	if (!trace_probe_log.argv)
181		return;
182
183	/* Recalculate the length and allocate buffer */
184	for (i = 0; i < trace_probe_log.argc; i++) {
185		if (i == trace_probe_log.index)
186			pos = len;
187		len += strlen(trace_probe_log.argv[i]) + 1;
188	}
189	command = kzalloc(len, GFP_KERNEL);
190	if (!command)
191		return;
192
193	if (trace_probe_log.index >= trace_probe_log.argc) {
194		/**
195		 * Set the error position is next to the last arg + space.
196		 * Note that len includes the terminal null and the cursor
197		 * appears at pos + 1.
198		 */
199		pos = len;
200		offset = 0;
201	}
202
203	/* And make a command string from argv array */
204	p = command;
205	for (i = 0; i < trace_probe_log.argc; i++) {
206		len = strlen(trace_probe_log.argv[i]);
207		strcpy(p, trace_probe_log.argv[i]);
208		p[len] = ' ';
209		p += len + 1;
210	}
211	*(p - 1) = '\0';
212
213	tracing_log_err(NULL, trace_probe_log.subsystem, command,
214			trace_probe_err_text, err_type, pos + offset);
215
216	kfree(command);
217}
218
219/* Split symbol and offset. */
220int traceprobe_split_symbol_offset(char *symbol, long *offset)
221{
222	char *tmp;
223	int ret;
224
225	if (!offset)
226		return -EINVAL;
227
228	tmp = strpbrk(symbol, "+-");
229	if (tmp) {
230		ret = kstrtol(tmp, 0, offset);
231		if (ret)
232			return ret;
233		*tmp = '\0';
234	} else
235		*offset = 0;
236
237	return 0;
238}
239
240/* @buf must has MAX_EVENT_NAME_LEN size */
241int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
242				char *buf, int offset)
243{
244	const char *slash, *event = *pevent;
245	int len;
246
247	slash = strchr(event, '/');
248	if (!slash)
249		slash = strchr(event, '.');
250
251	if (slash) {
252		if (slash == event) {
253			trace_probe_log_err(offset, NO_GROUP_NAME);
254			return -EINVAL;
255		}
256		if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
257			trace_probe_log_err(offset, GROUP_TOO_LONG);
258			return -EINVAL;
259		}
260		strscpy(buf, event, slash - event + 1);
261		if (!is_good_system_name(buf)) {
262			trace_probe_log_err(offset, BAD_GROUP_NAME);
263			return -EINVAL;
264		}
265		*pgroup = buf;
266		*pevent = slash + 1;
267		offset += slash - event + 1;
268		event = *pevent;
269	}
270	len = strlen(event);
271	if (len == 0) {
272		if (slash) {
273			*pevent = NULL;
274			return 0;
275		}
276		trace_probe_log_err(offset, NO_EVENT_NAME);
277		return -EINVAL;
278	} else if (len > MAX_EVENT_NAME_LEN) {
279		trace_probe_log_err(offset, EVENT_TOO_LONG);
280		return -EINVAL;
281	}
282	if (!is_good_name(event)) {
283		trace_probe_log_err(offset, BAD_EVENT_NAME);
284		return -EINVAL;
285	}
286	return 0;
287}
288
289static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
290				 struct traceprobe_parse_context *ctx)
291{
292	struct ftrace_event_field *field;
293	struct list_head *head;
294
295	head = trace_get_fields(ctx->event);
296	list_for_each_entry(field, head, link) {
297		if (!strcmp(arg, field->name)) {
298			code->op = FETCH_OP_TP_ARG;
299			code->data = field;
300			return 0;
301		}
302	}
303	return -ENOENT;
304}
305
306#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
307
308static u32 btf_type_int(const struct btf_type *t)
309{
310	return *(u32 *)(t + 1);
311}
312
313static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
314{
315	const struct btf_type *real_type;
316	u32 intdata;
317	s32 tid;
318
319	real_type = btf_type_skip_modifiers(btf, type->type, &tid);
320	if (!real_type)
321		return false;
322
323	if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
324		return false;
325
326	intdata = btf_type_int(real_type);
327	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
328		&& BTF_INT_BITS(intdata) == 8;
329}
330
331static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
332{
333	const struct btf_type *real_type;
334	const struct btf_array *array;
335	u32 intdata;
336	s32 tid;
337
338	if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
339		return false;
340
341	array = (const struct btf_array *)(type + 1);
342
343	real_type = btf_type_skip_modifiers(btf, array->type, &tid);
344
345	intdata = btf_type_int(real_type);
346	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
347		&& BTF_INT_BITS(intdata) == 8;
348}
349
350static int check_prepare_btf_string_fetch(char *typename,
351				struct fetch_insn **pcode,
352				struct traceprobe_parse_context *ctx)
353{
354	struct btf *btf = ctx->btf;
355
356	if (!btf || !ctx->last_type)
357		return 0;
358
359	/* char [] does not need any change. */
360	if (btf_type_is_char_array(btf, ctx->last_type))
361		return 0;
362
363	/* char * requires dereference the pointer. */
364	if (btf_type_is_char_ptr(btf, ctx->last_type)) {
365		struct fetch_insn *code = *pcode + 1;
366
367		if (code->op == FETCH_OP_END) {
368			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
369			return -E2BIG;
370		}
371		if (typename[0] == 'u')
372			code->op = FETCH_OP_UDEREF;
373		else
374			code->op = FETCH_OP_DEREF;
375		code->offset = 0;
376		*pcode = code;
377		return 0;
378	}
379	/* Other types are not available for string */
380	trace_probe_log_err(ctx->offset, BAD_TYPE4STR);
381	return -EINVAL;
382}
383
384static const char *fetch_type_from_btf_type(struct btf *btf,
385					const struct btf_type *type,
386					struct traceprobe_parse_context *ctx)
387{
388	u32 intdata;
389
390	/* TODO: const char * could be converted as a string */
391	switch (BTF_INFO_KIND(type->info)) {
392	case BTF_KIND_ENUM:
393		/* enum is "int", so convert to "s32" */
394		return "s32";
395	case BTF_KIND_ENUM64:
396		return "s64";
397	case BTF_KIND_PTR:
398		/* pointer will be converted to "x??" */
399		if (IS_ENABLED(CONFIG_64BIT))
400			return "x64";
401		else
402			return "x32";
403	case BTF_KIND_INT:
404		intdata = btf_type_int(type);
405		if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
406			switch (BTF_INT_BITS(intdata)) {
407			case 8:
408				return "s8";
409			case 16:
410				return "s16";
411			case 32:
412				return "s32";
413			case 64:
414				return "s64";
415			}
416		} else {	/* unsigned */
417			switch (BTF_INT_BITS(intdata)) {
418			case 8:
419				return "u8";
420			case 16:
421				return "u16";
422			case 32:
423				return "u32";
424			case 64:
425				return "u64";
426			}
427			/* bitfield, size is encoded in the type */
428			ctx->last_bitsize = BTF_INT_BITS(intdata);
429			ctx->last_bitoffs += BTF_INT_OFFSET(intdata);
430			return "u64";
431		}
432	}
433	/* TODO: support other types */
434
435	return NULL;
436}
437
438static int query_btf_context(struct traceprobe_parse_context *ctx)
439{
440	const struct btf_param *param;
441	const struct btf_type *type;
442	struct btf *btf;
443	s32 nr;
444
445	if (ctx->btf)
446		return 0;
447
448	if (!ctx->funcname)
449		return -EINVAL;
450
451	type = btf_find_func_proto(ctx->funcname, &btf);
452	if (!type)
453		return -ENOENT;
454
455	ctx->btf = btf;
456	ctx->proto = type;
457
458	/* ctx->params is optional, since func(void) will not have params. */
459	nr = 0;
460	param = btf_get_func_param(type, &nr);
461	if (!IS_ERR_OR_NULL(param)) {
462		/* Hide the first 'data' argument of tracepoint */
463		if (ctx->flags & TPARG_FL_TPOINT) {
464			nr--;
465			param++;
466		}
467	}
468
469	if (nr > 0) {
470		ctx->nr_params = nr;
471		ctx->params = param;
472	} else {
473		ctx->nr_params = 0;
474		ctx->params = NULL;
475	}
476
477	return 0;
478}
479
480static void clear_btf_context(struct traceprobe_parse_context *ctx)
481{
482	if (ctx->btf) {
483		btf_put(ctx->btf);
484		ctx->btf = NULL;
485		ctx->proto = NULL;
486		ctx->params = NULL;
487		ctx->nr_params = 0;
488	}
489}
490
491/* Return 1 if the field separater is arrow operator ('->') */
492static int split_next_field(char *varname, char **next_field,
493			    struct traceprobe_parse_context *ctx)
494{
495	char *field;
496	int ret = 0;
497
498	field = strpbrk(varname, ".-");
499	if (field) {
500		if (field[0] == '-' && field[1] == '>') {
501			field[0] = '\0';
502			field += 2;
503			ret = 1;
504		} else if (field[0] == '.') {
505			field[0] = '\0';
506			field += 1;
507		} else {
508			trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN);
509			return -EINVAL;
510		}
511		*next_field = field;
512	}
513
514	return ret;
515}
516
517/*
518 * Parse the field of data structure. The @type must be a pointer type
519 * pointing the target data structure type.
520 */
521static int parse_btf_field(char *fieldname, const struct btf_type *type,
522			   struct fetch_insn **pcode, struct fetch_insn *end,
523			   struct traceprobe_parse_context *ctx)
524{
525	struct fetch_insn *code = *pcode;
526	const struct btf_member *field;
527	u32 bitoffs, anon_offs;
528	char *next;
529	int is_ptr;
530	s32 tid;
531
532	do {
533		/* Outer loop for solving arrow operator ('->') */
534		if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
535			trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
536			return -EINVAL;
537		}
538		/* Convert a struct pointer type to a struct type */
539		type = btf_type_skip_modifiers(ctx->btf, type->type, &tid);
540		if (!type) {
541			trace_probe_log_err(ctx->offset, BAD_BTF_TID);
542			return -EINVAL;
543		}
544
545		bitoffs = 0;
546		do {
547			/* Inner loop for solving dot operator ('.') */
548			next = NULL;
549			is_ptr = split_next_field(fieldname, &next, ctx);
550			if (is_ptr < 0)
551				return is_ptr;
552
553			anon_offs = 0;
554			field = btf_find_struct_member(ctx->btf, type, fieldname,
555						       &anon_offs);
556			if (!field) {
557				trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
558				return -ENOENT;
559			}
560			/* Add anonymous structure/union offset */
561			bitoffs += anon_offs;
562
563			/* Accumulate the bit-offsets of the dot-connected fields */
564			if (btf_type_kflag(type)) {
565				bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
566				ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
567			} else {
568				bitoffs += field->offset;
569				ctx->last_bitsize = 0;
570			}
571
572			type = btf_type_skip_modifiers(ctx->btf, field->type, &tid);
573			if (!type) {
574				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
575				return -EINVAL;
576			}
577
578			ctx->offset += next - fieldname;
579			fieldname = next;
580		} while (!is_ptr && fieldname);
581
582		if (++code == end) {
583			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
584			return -EINVAL;
585		}
586		code->op = FETCH_OP_DEREF;	/* TODO: user deref support */
587		code->offset = bitoffs / 8;
588		*pcode = code;
589
590		ctx->last_bitoffs = bitoffs % 8;
591		ctx->last_type = type;
592	} while (fieldname);
593
594	return 0;
595}
596
597static int __store_entry_arg(struct trace_probe *tp, int argnum);
598
599static int parse_btf_arg(char *varname,
600			 struct fetch_insn **pcode, struct fetch_insn *end,
601			 struct traceprobe_parse_context *ctx)
602{
603	struct fetch_insn *code = *pcode;
604	const struct btf_param *params;
605	const struct btf_type *type;
606	char *field = NULL;
607	int i, is_ptr, ret;
608	u32 tid;
609
610	if (WARN_ON_ONCE(!ctx->funcname))
611		return -EINVAL;
612
613	is_ptr = split_next_field(varname, &field, ctx);
614	if (is_ptr < 0)
615		return is_ptr;
616	if (!is_ptr && field) {
617		/* dot-connected field on an argument is not supported. */
618		trace_probe_log_err(ctx->offset + field - varname,
619				    NOSUP_DAT_ARG);
620		return -EOPNOTSUPP;
621	}
622
623	if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
624		code->op = FETCH_OP_RETVAL;
625		/* Check whether the function return type is not void */
626		if (query_btf_context(ctx) == 0) {
627			if (ctx->proto->type == 0) {
628				trace_probe_log_err(ctx->offset, NO_RETVAL);
629				return -ENOENT;
630			}
631			tid = ctx->proto->type;
632			goto found;
633		}
634		if (field) {
635			trace_probe_log_err(ctx->offset + field - varname,
636					    NO_BTF_ENTRY);
637			return -ENOENT;
638		}
639		return 0;
640	}
641
642	if (!ctx->btf) {
643		ret = query_btf_context(ctx);
644		if (ret < 0 || ctx->nr_params == 0) {
645			trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
646			return PTR_ERR(params);
647		}
648	}
649	params = ctx->params;
650
651	for (i = 0; i < ctx->nr_params; i++) {
652		const char *name = btf_name_by_offset(ctx->btf, params[i].name_off);
653
654		if (name && !strcmp(name, varname)) {
655			if (tparg_is_function_entry(ctx->flags)) {
656				code->op = FETCH_OP_ARG;
657				if (ctx->flags & TPARG_FL_TPOINT)
658					code->param = i + 1;
659				else
660					code->param = i;
661			} else if (tparg_is_function_return(ctx->flags)) {
662				code->op = FETCH_OP_EDATA;
663				ret = __store_entry_arg(ctx->tp, i);
664				if (ret < 0) {
665					/* internal error */
666					return ret;
667				}
668				code->offset = ret;
669			}
670			tid = params[i].type;
671			goto found;
672		}
673	}
674	trace_probe_log_err(ctx->offset, NO_BTFARG);
675	return -ENOENT;
676
677found:
678	type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
679	if (!type) {
680		trace_probe_log_err(ctx->offset, BAD_BTF_TID);
681		return -EINVAL;
682	}
683	/* Initialize the last type information */
684	ctx->last_type = type;
685	ctx->last_bitoffs = 0;
686	ctx->last_bitsize = 0;
687	if (field) {
688		ctx->offset += field - varname;
689		return parse_btf_field(field, type, pcode, end, ctx);
690	}
691	return 0;
692}
693
694static const struct fetch_type *find_fetch_type_from_btf_type(
695					struct traceprobe_parse_context *ctx)
696{
697	struct btf *btf = ctx->btf;
698	const char *typestr = NULL;
699
700	if (btf && ctx->last_type)
701		typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx);
702
703	return find_fetch_type(typestr, ctx->flags);
704}
705
706static int parse_btf_bitfield(struct fetch_insn **pcode,
707			      struct traceprobe_parse_context *ctx)
708{
709	struct fetch_insn *code = *pcode;
710
711	if ((ctx->last_bitsize % 8 == 0) && ctx->last_bitoffs == 0)
712		return 0;
713
714	code++;
715	if (code->op != FETCH_OP_NOP) {
716		trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
717		return -EINVAL;
718	}
719	*pcode = code;
720
721	code->op = FETCH_OP_MOD_BF;
722	code->lshift = 64 - (ctx->last_bitsize + ctx->last_bitoffs);
723	code->rshift = 64 - ctx->last_bitsize;
724	code->basesize = 64 / 8;
725	return 0;
726}
727
728#else
729static void clear_btf_context(struct traceprobe_parse_context *ctx)
730{
731	ctx->btf = NULL;
732}
733
734static int query_btf_context(struct traceprobe_parse_context *ctx)
735{
736	return -EOPNOTSUPP;
737}
738
739static int parse_btf_arg(char *varname,
740			 struct fetch_insn **pcode, struct fetch_insn *end,
741			 struct traceprobe_parse_context *ctx)
742{
743	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
744	return -EOPNOTSUPP;
745}
746
747static int parse_btf_bitfield(struct fetch_insn **pcode,
748			      struct traceprobe_parse_context *ctx)
749{
750	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
751	return -EOPNOTSUPP;
752}
753
754#define find_fetch_type_from_btf_type(ctx)		\
755	find_fetch_type(NULL, ctx->flags)
756
757static int check_prepare_btf_string_fetch(char *typename,
758				struct fetch_insn **pcode,
759				struct traceprobe_parse_context *ctx)
760{
761	return 0;
762}
763
764#endif
765
766#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
767
768static int __store_entry_arg(struct trace_probe *tp, int argnum)
769{
770	struct probe_entry_arg *earg = tp->entry_arg;
771	bool match = false;
772	int i, offset;
773
774	if (!earg) {
775		earg = kzalloc(sizeof(*tp->entry_arg), GFP_KERNEL);
776		if (!earg)
777			return -ENOMEM;
778		earg->size = 2 * tp->nr_args + 1;
779		earg->code = kcalloc(earg->size, sizeof(struct fetch_insn),
780				     GFP_KERNEL);
781		if (!earg->code) {
782			kfree(earg);
783			return -ENOMEM;
784		}
785		/* Fill the code buffer with 'end' to simplify it */
786		for (i = 0; i < earg->size; i++)
787			earg->code[i].op = FETCH_OP_END;
788		tp->entry_arg = earg;
789	}
790
791	offset = 0;
792	for (i = 0; i < earg->size - 1; i++) {
793		switch (earg->code[i].op) {
794		case FETCH_OP_END:
795			earg->code[i].op = FETCH_OP_ARG;
796			earg->code[i].param = argnum;
797			earg->code[i + 1].op = FETCH_OP_ST_EDATA;
798			earg->code[i + 1].offset = offset;
799			return offset;
800		case FETCH_OP_ARG:
801			match = (earg->code[i].param == argnum);
802			break;
803		case FETCH_OP_ST_EDATA:
804			offset = earg->code[i].offset;
805			if (match)
806				return offset;
807			offset += sizeof(unsigned long);
808			break;
809		default:
810			break;
811		}
812	}
813	return -ENOSPC;
814}
815
816int traceprobe_get_entry_data_size(struct trace_probe *tp)
817{
818	struct probe_entry_arg *earg = tp->entry_arg;
819	int i, size = 0;
820
821	if (!earg)
822		return 0;
823
824	for (i = 0; i < earg->size; i++) {
825		switch (earg->code[i].op) {
826		case FETCH_OP_END:
827			goto out;
828		case FETCH_OP_ST_EDATA:
829			size = earg->code[i].offset + sizeof(unsigned long);
830			break;
831		default:
832			break;
833		}
834	}
835out:
836	return size;
837}
838
839void store_trace_entry_data(void *edata, struct trace_probe *tp, struct pt_regs *regs)
840{
841	struct probe_entry_arg *earg = tp->entry_arg;
842	unsigned long val = 0;
843	int i;
844
845	if (!earg)
846		return;
847
848	for (i = 0; i < earg->size; i++) {
849		struct fetch_insn *code = &earg->code[i];
850
851		switch (code->op) {
852		case FETCH_OP_ARG:
853			val = regs_get_kernel_argument(regs, code->param);
854			break;
855		case FETCH_OP_ST_EDATA:
856			*(unsigned long *)((unsigned long)edata + code->offset) = val;
857			break;
858		case FETCH_OP_END:
859			goto end;
860		default:
861			break;
862		}
863	}
864end:
865	return;
866}
867NOKPROBE_SYMBOL(store_trace_entry_data)
868#endif
869
870#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
871
872/* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
873static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
874			    struct fetch_insn **pcode,
875			    struct fetch_insn *end,
876			    struct traceprobe_parse_context *ctx)
877{
878	struct fetch_insn *code = *pcode;
879	int err = TP_ERR_BAD_VAR;
880	char *arg = orig_arg + 1;
881	unsigned long param;
882	int ret = 0;
883	int len;
884
885	if (ctx->flags & TPARG_FL_TEVENT) {
886		if (code->data)
887			return -EFAULT;
888		ret = parse_trace_event_arg(arg, code, ctx);
889		if (!ret)
890			return 0;
891		if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
892			code->op = FETCH_OP_COMM;
893			return 0;
894		}
895		/* backward compatibility */
896		ctx->offset = 0;
897		goto inval;
898	}
899
900	if (str_has_prefix(arg, "retval")) {
901		if (!(ctx->flags & TPARG_FL_RETURN)) {
902			err = TP_ERR_RETVAL_ON_PROBE;
903			goto inval;
904		}
905		if (!(ctx->flags & TPARG_FL_KERNEL) ||
906		    !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
907			code->op = FETCH_OP_RETVAL;
908			return 0;
909		}
910		return parse_btf_arg(orig_arg, pcode, end, ctx);
911	}
912
913	len = str_has_prefix(arg, "stack");
914	if (len) {
915
916		if (arg[len] == '\0') {
917			code->op = FETCH_OP_STACKP;
918			return 0;
919		}
920
921		if (isdigit(arg[len])) {
922			ret = kstrtoul(arg + len, 10, &param);
923			if (ret)
924				goto inval;
925
926			if ((ctx->flags & TPARG_FL_KERNEL) &&
927			    param > PARAM_MAX_STACK) {
928				err = TP_ERR_BAD_STACK_NUM;
929				goto inval;
930			}
931			code->op = FETCH_OP_STACK;
932			code->param = (unsigned int)param;
933			return 0;
934		}
935		goto inval;
936	}
937
938	if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
939		code->op = FETCH_OP_COMM;
940		return 0;
941	}
942
943#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
944	len = str_has_prefix(arg, "arg");
945	if (len) {
946		ret = kstrtoul(arg + len, 10, &param);
947		if (ret)
948			goto inval;
949
950		if (!param || param > PARAM_MAX_STACK) {
951			err = TP_ERR_BAD_ARG_NUM;
952			goto inval;
953		}
954		param--; /* argN starts from 1, but internal arg[N] starts from 0 */
955
956		if (tparg_is_function_entry(ctx->flags)) {
957			code->op = FETCH_OP_ARG;
958			code->param = (unsigned int)param;
959			/*
960			 * The tracepoint probe will probe a stub function, and the
961			 * first parameter of the stub is a dummy and should be ignored.
962			 */
963			if (ctx->flags & TPARG_FL_TPOINT)
964				code->param++;
965		} else if (tparg_is_function_return(ctx->flags)) {
966			/* function entry argument access from return probe */
967			ret = __store_entry_arg(ctx->tp, param);
968			if (ret < 0)	/* This error should be an internal error */
969				return ret;
970
971			code->op = FETCH_OP_EDATA;
972			code->offset = ret;
973		} else {
974			err = TP_ERR_NOFENTRY_ARGS;
975			goto inval;
976		}
977		return 0;
978	}
979#endif
980
981inval:
982	__trace_probe_log_err(ctx->offset, err);
983	return -EINVAL;
984}
985
986static int str_to_immediate(char *str, unsigned long *imm)
987{
988	if (isdigit(str[0]))
989		return kstrtoul(str, 0, imm);
990	else if (str[0] == '-')
991		return kstrtol(str, 0, (long *)imm);
992	else if (str[0] == '+')
993		return kstrtol(str + 1, 0, (long *)imm);
994	return -EINVAL;
995}
996
997static int __parse_imm_string(char *str, char **pbuf, int offs)
998{
999	size_t len = strlen(str);
1000
1001	if (str[len - 1] != '"') {
1002		trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
1003		return -EINVAL;
1004	}
1005	*pbuf = kstrndup(str, len - 1, GFP_KERNEL);
1006	if (!*pbuf)
1007		return -ENOMEM;
1008	return 0;
1009}
1010
1011/* Recursive argument parser */
1012static int
1013parse_probe_arg(char *arg, const struct fetch_type *type,
1014		struct fetch_insn **pcode, struct fetch_insn *end,
1015		struct traceprobe_parse_context *ctx)
1016{
1017	struct fetch_insn *code = *pcode;
1018	unsigned long param;
1019	int deref = FETCH_OP_DEREF;
1020	long offset = 0;
1021	char *tmp;
1022	int ret = 0;
1023
1024	switch (arg[0]) {
1025	case '$':
1026		ret = parse_probe_vars(arg, type, pcode, end, ctx);
1027		break;
1028
1029	case '%':	/* named register */
1030		if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
1031			/* eprobe and fprobe do not handle registers */
1032			trace_probe_log_err(ctx->offset, BAD_VAR);
1033			break;
1034		}
1035		ret = regs_query_register_offset(arg + 1);
1036		if (ret >= 0) {
1037			code->op = FETCH_OP_REG;
1038			code->param = (unsigned int)ret;
1039			ret = 0;
1040		} else
1041			trace_probe_log_err(ctx->offset, BAD_REG_NAME);
1042		break;
1043
1044	case '@':	/* memory, file-offset or symbol */
1045		if (isdigit(arg[1])) {
1046			ret = kstrtoul(arg + 1, 0, &param);
1047			if (ret) {
1048				trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
1049				break;
1050			}
1051			/* load address */
1052			code->op = FETCH_OP_IMM;
1053			code->immediate = param;
1054		} else if (arg[1] == '+') {
1055			/* kprobes don't support file offsets */
1056			if (ctx->flags & TPARG_FL_KERNEL) {
1057				trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
1058				return -EINVAL;
1059			}
1060			ret = kstrtol(arg + 2, 0, &offset);
1061			if (ret) {
1062				trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
1063				break;
1064			}
1065
1066			code->op = FETCH_OP_FOFFS;
1067			code->immediate = (unsigned long)offset;  // imm64?
1068		} else {
1069			/* uprobes don't support symbols */
1070			if (!(ctx->flags & TPARG_FL_KERNEL)) {
1071				trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
1072				return -EINVAL;
1073			}
1074			/* Preserve symbol for updating */
1075			code->op = FETCH_NOP_SYMBOL;
1076			code->data = kstrdup(arg + 1, GFP_KERNEL);
1077			if (!code->data)
1078				return -ENOMEM;
1079			if (++code == end) {
1080				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1081				return -EINVAL;
1082			}
1083			code->op = FETCH_OP_IMM;
1084			code->immediate = 0;
1085		}
1086		/* These are fetching from memory */
1087		if (++code == end) {
1088			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1089			return -EINVAL;
1090		}
1091		*pcode = code;
1092		code->op = FETCH_OP_DEREF;
1093		code->offset = offset;
1094		break;
1095
1096	case '+':	/* deref memory */
1097	case '-':
1098		if (arg[1] == 'u') {
1099			deref = FETCH_OP_UDEREF;
1100			arg[1] = arg[0];
1101			arg++;
1102		}
1103		if (arg[0] == '+')
1104			arg++;	/* Skip '+', because kstrtol() rejects it. */
1105		tmp = strchr(arg, '(');
1106		if (!tmp) {
1107			trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
1108			return -EINVAL;
1109		}
1110		*tmp = '\0';
1111		ret = kstrtol(arg, 0, &offset);
1112		if (ret) {
1113			trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
1114			break;
1115		}
1116		ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
1117		arg = tmp + 1;
1118		tmp = strrchr(arg, ')');
1119		if (!tmp) {
1120			trace_probe_log_err(ctx->offset + strlen(arg),
1121					    DEREF_OPEN_BRACE);
1122			return -EINVAL;
1123		} else {
1124			const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags);
1125			int cur_offs = ctx->offset;
1126
1127			*tmp = '\0';
1128			ret = parse_probe_arg(arg, t2, &code, end, ctx);
1129			if (ret)
1130				break;
1131			ctx->offset = cur_offs;
1132			if (code->op == FETCH_OP_COMM ||
1133			    code->op == FETCH_OP_DATA) {
1134				trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
1135				return -EINVAL;
1136			}
1137			if (++code == end) {
1138				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1139				return -EINVAL;
1140			}
1141			*pcode = code;
1142
1143			code->op = deref;
1144			code->offset = offset;
1145			/* Reset the last type if used */
1146			ctx->last_type = NULL;
1147		}
1148		break;
1149	case '\\':	/* Immediate value */
1150		if (arg[1] == '"') {	/* Immediate string */
1151			ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
1152			if (ret)
1153				break;
1154			code->op = FETCH_OP_DATA;
1155			code->data = tmp;
1156		} else {
1157			ret = str_to_immediate(arg + 1, &code->immediate);
1158			if (ret)
1159				trace_probe_log_err(ctx->offset + 1, BAD_IMM);
1160			else
1161				code->op = FETCH_OP_IMM;
1162		}
1163		break;
1164	default:
1165		if (isalpha(arg[0]) || arg[0] == '_') {	/* BTF variable */
1166			if (!tparg_is_function_entry(ctx->flags) &&
1167			    !tparg_is_function_return(ctx->flags)) {
1168				trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1169				return -EINVAL;
1170			}
1171			ret = parse_btf_arg(arg, pcode, end, ctx);
1172			break;
1173		}
1174	}
1175	if (!ret && code->op == FETCH_OP_NOP) {
1176		/* Parsed, but do not find fetch method */
1177		trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
1178		ret = -EINVAL;
1179	}
1180	return ret;
1181}
1182
1183#define BYTES_TO_BITS(nb)	((BITS_PER_LONG * (nb)) / sizeof(long))
1184
1185/* Bitfield type needs to be parsed into a fetch function */
1186static int __parse_bitfield_probe_arg(const char *bf,
1187				      const struct fetch_type *t,
1188				      struct fetch_insn **pcode)
1189{
1190	struct fetch_insn *code = *pcode;
1191	unsigned long bw, bo;
1192	char *tail;
1193
1194	if (*bf != 'b')
1195		return 0;
1196
1197	bw = simple_strtoul(bf + 1, &tail, 0);	/* Use simple one */
1198
1199	if (bw == 0 || *tail != '@')
1200		return -EINVAL;
1201
1202	bf = tail + 1;
1203	bo = simple_strtoul(bf, &tail, 0);
1204
1205	if (tail == bf || *tail != '/')
1206		return -EINVAL;
1207	code++;
1208	if (code->op != FETCH_OP_NOP)
1209		return -EINVAL;
1210	*pcode = code;
1211
1212	code->op = FETCH_OP_MOD_BF;
1213	code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
1214	code->rshift = BYTES_TO_BITS(t->size) - bw;
1215	code->basesize = t->size;
1216
1217	return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
1218}
1219
1220/* Split type part from @arg and return it. */
1221static char *parse_probe_arg_type(char *arg, struct probe_arg *parg,
1222				  struct traceprobe_parse_context *ctx)
1223{
1224	char *t = NULL, *t2, *t3;
1225	int offs;
1226
1227	t = strchr(arg, ':');
1228	if (t) {
1229		*t++ = '\0';
1230		t2 = strchr(t, '[');
1231		if (t2) {
1232			*t2++ = '\0';
1233			t3 = strchr(t2, ']');
1234			if (!t3) {
1235				offs = t2 + strlen(t2) - arg;
1236
1237				trace_probe_log_err(ctx->offset + offs,
1238						    ARRAY_NO_CLOSE);
1239				return ERR_PTR(-EINVAL);
1240			} else if (t3[1] != '\0') {
1241				trace_probe_log_err(ctx->offset + t3 + 1 - arg,
1242						    BAD_ARRAY_SUFFIX);
1243				return ERR_PTR(-EINVAL);
1244			}
1245			*t3 = '\0';
1246			if (kstrtouint(t2, 0, &parg->count) || !parg->count) {
1247				trace_probe_log_err(ctx->offset + t2 - arg,
1248						    BAD_ARRAY_NUM);
1249				return ERR_PTR(-EINVAL);
1250			}
1251			if (parg->count > MAX_ARRAY_LEN) {
1252				trace_probe_log_err(ctx->offset + t2 - arg,
1253						    ARRAY_TOO_BIG);
1254				return ERR_PTR(-EINVAL);
1255			}
1256		}
1257	}
1258	offs = t ? t - arg : 0;
1259
1260	/*
1261	 * Since $comm and immediate string can not be dereferenced,
1262	 * we can find those by strcmp. But ignore for eprobes.
1263	 */
1264	if (!(ctx->flags & TPARG_FL_TEVENT) &&
1265	    (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
1266	     strncmp(arg, "\\\"", 2) == 0)) {
1267		/* The type of $comm must be "string", and not an array type. */
1268		if (parg->count || (t && strcmp(t, "string"))) {
1269			trace_probe_log_err(ctx->offset + offs, NEED_STRING_TYPE);
1270			return ERR_PTR(-EINVAL);
1271		}
1272		parg->type = find_fetch_type("string", ctx->flags);
1273	} else
1274		parg->type = find_fetch_type(t, ctx->flags);
1275
1276	if (!parg->type) {
1277		trace_probe_log_err(ctx->offset + offs, BAD_TYPE);
1278		return ERR_PTR(-EINVAL);
1279	}
1280
1281	return t;
1282}
1283
1284/* After parsing, adjust the fetch_insn according to the probe_arg */
1285static int finalize_fetch_insn(struct fetch_insn *code,
1286			       struct probe_arg *parg,
1287			       char *type,
1288			       int type_offset,
1289			       struct traceprobe_parse_context *ctx)
1290{
1291	struct fetch_insn *scode;
1292	int ret;
1293
1294	/* Store operation */
1295	if (parg->type->is_string) {
1296		/* Check bad combination of the type and the last fetch_insn. */
1297		if (!strcmp(parg->type->name, "symstr")) {
1298			if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
1299			    code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
1300			    code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
1301				trace_probe_log_err(ctx->offset + type_offset,
1302						    BAD_SYMSTRING);
1303				return -EINVAL;
1304			}
1305		} else {
1306			if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
1307			    code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
1308			    code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
1309				trace_probe_log_err(ctx->offset + type_offset,
1310						    BAD_STRING);
1311				return -EINVAL;
1312			}
1313		}
1314
1315		if (!strcmp(parg->type->name, "symstr") ||
1316		    (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
1317		     code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
1318		     parg->count) {
1319			/*
1320			 * IMM, DATA and COMM is pointing actual address, those
1321			 * must be kept, and if parg->count != 0, this is an
1322			 * array of string pointers instead of string address
1323			 * itself.
1324			 * For the symstr, it doesn't need to dereference, thus
1325			 * it just get the value.
1326			 */
1327			code++;
1328			if (code->op != FETCH_OP_NOP) {
1329				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1330				return -EINVAL;
1331			}
1332		}
1333
1334		/* If op == DEREF, replace it with STRING */
1335		if (!strcmp(parg->type->name, "ustring") ||
1336		    code->op == FETCH_OP_UDEREF)
1337			code->op = FETCH_OP_ST_USTRING;
1338		else if (!strcmp(parg->type->name, "symstr"))
1339			code->op = FETCH_OP_ST_SYMSTR;
1340		else
1341			code->op = FETCH_OP_ST_STRING;
1342		code->size = parg->type->size;
1343		parg->dynamic = true;
1344	} else if (code->op == FETCH_OP_DEREF) {
1345		code->op = FETCH_OP_ST_MEM;
1346		code->size = parg->type->size;
1347	} else if (code->op == FETCH_OP_UDEREF) {
1348		code->op = FETCH_OP_ST_UMEM;
1349		code->size = parg->type->size;
1350	} else {
1351		code++;
1352		if (code->op != FETCH_OP_NOP) {
1353			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1354			return -E2BIG;
1355		}
1356		code->op = FETCH_OP_ST_RAW;
1357		code->size = parg->type->size;
1358	}
1359
1360	/* Save storing fetch_insn. */
1361	scode = code;
1362
1363	/* Modify operation */
1364	if (type != NULL) {
1365		/* Bitfield needs a special fetch_insn. */
1366		ret = __parse_bitfield_probe_arg(type, parg->type, &code);
1367		if (ret) {
1368			trace_probe_log_err(ctx->offset + type_offset, BAD_BITFIELD);
1369			return ret;
1370		}
1371	} else if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1372		   ctx->last_type) {
1373		/* If user not specified the type, try parsing BTF bitfield. */
1374		ret = parse_btf_bitfield(&code, ctx);
1375		if (ret)
1376			return ret;
1377	}
1378
1379	/* Loop(Array) operation */
1380	if (parg->count) {
1381		if (scode->op != FETCH_OP_ST_MEM &&
1382		    scode->op != FETCH_OP_ST_STRING &&
1383		    scode->op != FETCH_OP_ST_USTRING) {
1384			trace_probe_log_err(ctx->offset + type_offset, BAD_STRING);
1385			return -EINVAL;
1386		}
1387		code++;
1388		if (code->op != FETCH_OP_NOP) {
1389			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1390			return -E2BIG;
1391		}
1392		code->op = FETCH_OP_LP_ARRAY;
1393		code->param = parg->count;
1394	}
1395
1396	/* Finalize the fetch_insn array. */
1397	code++;
1398	code->op = FETCH_OP_END;
1399
1400	return 0;
1401}
1402
1403/* String length checking wrapper */
1404static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
1405					   struct probe_arg *parg,
1406					   struct traceprobe_parse_context *ctx)
1407{
1408	struct fetch_insn *code, *tmp = NULL;
1409	char *type, *arg;
1410	int ret, len;
1411
1412	len = strlen(argv);
1413	if (len > MAX_ARGSTR_LEN) {
1414		trace_probe_log_err(ctx->offset, ARG_TOO_LONG);
1415		return -E2BIG;
1416	} else if (len == 0) {
1417		trace_probe_log_err(ctx->offset, NO_ARG_BODY);
1418		return -EINVAL;
1419	}
1420
1421	arg = kstrdup(argv, GFP_KERNEL);
1422	if (!arg)
1423		return -ENOMEM;
1424
1425	parg->comm = kstrdup(arg, GFP_KERNEL);
1426	if (!parg->comm) {
1427		ret = -ENOMEM;
1428		goto out;
1429	}
1430
1431	type = parse_probe_arg_type(arg, parg, ctx);
1432	if (IS_ERR(type)) {
1433		ret = PTR_ERR(type);
1434		goto out;
1435	}
1436
1437	code = tmp = kcalloc(FETCH_INSN_MAX, sizeof(*code), GFP_KERNEL);
1438	if (!code) {
1439		ret = -ENOMEM;
1440		goto out;
1441	}
1442	code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
1443
1444	ctx->last_type = NULL;
1445	ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
1446			      ctx);
1447	if (ret < 0)
1448		goto fail;
1449
1450	/* Update storing type if BTF is available */
1451	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1452	    ctx->last_type) {
1453		if (!type) {
1454			parg->type = find_fetch_type_from_btf_type(ctx);
1455		} else if (strstr(type, "string")) {
1456			ret = check_prepare_btf_string_fetch(type, &code, ctx);
1457			if (ret)
1458				goto fail;
1459		}
1460	}
1461	parg->offset = *size;
1462	*size += parg->type->size * (parg->count ?: 1);
1463
1464	if (parg->count) {
1465		len = strlen(parg->type->fmttype) + 6;
1466		parg->fmt = kmalloc(len, GFP_KERNEL);
1467		if (!parg->fmt) {
1468			ret = -ENOMEM;
1469			goto out;
1470		}
1471		snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
1472			 parg->count);
1473	}
1474
1475	ret = finalize_fetch_insn(code, parg, type, type ? type - arg : 0, ctx);
1476	if (ret < 0)
1477		goto fail;
1478
1479	for (; code < tmp + FETCH_INSN_MAX; code++)
1480		if (code->op == FETCH_OP_END)
1481			break;
1482	/* Shrink down the code buffer */
1483	parg->code = kcalloc(code - tmp + 1, sizeof(*code), GFP_KERNEL);
1484	if (!parg->code)
1485		ret = -ENOMEM;
1486	else
1487		memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
1488
1489fail:
1490	if (ret < 0) {
1491		for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
1492			if (code->op == FETCH_NOP_SYMBOL ||
1493			    code->op == FETCH_OP_DATA)
1494				kfree(code->data);
1495	}
1496	kfree(tmp);
1497out:
1498	kfree(arg);
1499
1500	return ret;
1501}
1502
1503/* Return 1 if name is reserved or already used by another argument */
1504static int traceprobe_conflict_field_name(const char *name,
1505					  struct probe_arg *args, int narg)
1506{
1507	int i;
1508
1509	for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
1510		if (strcmp(reserved_field_names[i], name) == 0)
1511			return 1;
1512
1513	for (i = 0; i < narg; i++)
1514		if (strcmp(args[i].name, name) == 0)
1515			return 1;
1516
1517	return 0;
1518}
1519
1520static char *generate_probe_arg_name(const char *arg, int idx)
1521{
1522	char *name = NULL;
1523	const char *end;
1524
1525	/*
1526	 * If argument name is omitted, try arg as a name (BTF variable)
1527	 * or "argN".
1528	 */
1529	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
1530		end = strchr(arg, ':');
1531		if (!end)
1532			end = arg + strlen(arg);
1533
1534		name = kmemdup_nul(arg, end - arg, GFP_KERNEL);
1535		if (!name || !is_good_name(name)) {
1536			kfree(name);
1537			name = NULL;
1538		}
1539	}
1540
1541	if (!name)
1542		name = kasprintf(GFP_KERNEL, "arg%d", idx + 1);
1543
1544	return name;
1545}
1546
1547int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
1548			       struct traceprobe_parse_context *ctx)
1549{
1550	struct probe_arg *parg = &tp->args[i];
1551	const char *body;
1552
1553	ctx->tp = tp;
1554	body = strchr(arg, '=');
1555	if (body) {
1556		if (body - arg > MAX_ARG_NAME_LEN) {
1557			trace_probe_log_err(0, ARG_NAME_TOO_LONG);
1558			return -EINVAL;
1559		} else if (body == arg) {
1560			trace_probe_log_err(0, NO_ARG_NAME);
1561			return -EINVAL;
1562		}
1563		parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
1564		body++;
1565	} else {
1566		parg->name = generate_probe_arg_name(arg, i);
1567		body = arg;
1568	}
1569	if (!parg->name)
1570		return -ENOMEM;
1571
1572	if (!is_good_name(parg->name)) {
1573		trace_probe_log_err(0, BAD_ARG_NAME);
1574		return -EINVAL;
1575	}
1576	if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
1577		trace_probe_log_err(0, USED_ARG_NAME);
1578		return -EINVAL;
1579	}
1580	ctx->offset = body - arg;
1581	/* Parse fetch argument */
1582	return traceprobe_parse_probe_arg_body(body, &tp->size, parg, ctx);
1583}
1584
1585void traceprobe_free_probe_arg(struct probe_arg *arg)
1586{
1587	struct fetch_insn *code = arg->code;
1588
1589	while (code && code->op != FETCH_OP_END) {
1590		if (code->op == FETCH_NOP_SYMBOL ||
1591		    code->op == FETCH_OP_DATA)
1592			kfree(code->data);
1593		code++;
1594	}
1595	kfree(arg->code);
1596	kfree(arg->name);
1597	kfree(arg->comm);
1598	kfree(arg->fmt);
1599}
1600
1601static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
1602			    struct traceprobe_parse_context *ctx)
1603{
1604	int i, found = 0;
1605
1606	for (i = 0; i < argc; i++)
1607		if (str_has_prefix(argv[i], "$arg")) {
1608			trace_probe_log_set_index(i + 2);
1609
1610			if (!tparg_is_function_entry(ctx->flags) &&
1611			    !tparg_is_function_return(ctx->flags)) {
1612				trace_probe_log_err(0, NOFENTRY_ARGS);
1613				return -EINVAL;
1614			}
1615
1616			if (isdigit(argv[i][4])) {
1617				found = 1;
1618				continue;
1619			}
1620
1621			if (argv[i][4] != '*') {
1622				trace_probe_log_err(0, BAD_VAR);
1623				return -EINVAL;
1624			}
1625
1626			if (*args_idx >= 0 && *args_idx < argc) {
1627				trace_probe_log_err(0, DOUBLE_ARGS);
1628				return -EINVAL;
1629			}
1630			found = 1;
1631			*args_idx = i;
1632		}
1633
1634	return found;
1635}
1636
1637static int sprint_nth_btf_arg(int idx, const char *type,
1638			      char *buf, int bufsize,
1639			      struct traceprobe_parse_context *ctx)
1640{
1641	const char *name;
1642	int ret;
1643
1644	if (idx >= ctx->nr_params) {
1645		trace_probe_log_err(0, NO_BTFARG);
1646		return -ENOENT;
1647	}
1648	name = btf_name_by_offset(ctx->btf, ctx->params[idx].name_off);
1649	if (!name) {
1650		trace_probe_log_err(0, NO_BTF_ENTRY);
1651		return -ENOENT;
1652	}
1653	ret = snprintf(buf, bufsize, "%s%s", name, type);
1654	if (ret >= bufsize) {
1655		trace_probe_log_err(0, ARGS_2LONG);
1656		return -E2BIG;
1657	}
1658	return ret;
1659}
1660
1661/* Return new_argv which must be freed after use */
1662const char **traceprobe_expand_meta_args(int argc, const char *argv[],
1663					 int *new_argc, char *buf, int bufsize,
1664					 struct traceprobe_parse_context *ctx)
1665{
1666	const struct btf_param *params = NULL;
1667	int i, j, n, used, ret, args_idx = -1;
1668	const char **new_argv = NULL;
1669
1670	ret = argv_has_var_arg(argc, argv, &args_idx, ctx);
1671	if (ret < 0)
1672		return ERR_PTR(ret);
1673
1674	if (!ret) {
1675		*new_argc = argc;
1676		return NULL;
1677	}
1678
1679	ret = query_btf_context(ctx);
1680	if (ret < 0 || ctx->nr_params == 0) {
1681		if (args_idx != -1) {
1682			/* $arg* requires BTF info */
1683			trace_probe_log_err(0, NOSUP_BTFARG);
1684			return (const char **)params;
1685		}
1686		*new_argc = argc;
1687		return NULL;
1688	}
1689
1690	if (args_idx >= 0)
1691		*new_argc = argc + ctx->nr_params - 1;
1692	else
1693		*new_argc = argc;
1694
1695	new_argv = kcalloc(*new_argc, sizeof(char *), GFP_KERNEL);
1696	if (!new_argv)
1697		return ERR_PTR(-ENOMEM);
1698
1699	used = 0;
1700	for (i = 0, j = 0; i < argc; i++) {
1701		trace_probe_log_set_index(i + 2);
1702		if (i == args_idx) {
1703			for (n = 0; n < ctx->nr_params; n++) {
1704				ret = sprint_nth_btf_arg(n, "", buf + used,
1705							 bufsize - used, ctx);
1706				if (ret < 0)
1707					goto error;
1708
1709				new_argv[j++] = buf + used;
1710				used += ret + 1;
1711			}
1712			continue;
1713		}
1714
1715		if (str_has_prefix(argv[i], "$arg")) {
1716			char *type = NULL;
1717
1718			n = simple_strtoul(argv[i] + 4, &type, 10);
1719			if (type && !(*type == ':' || *type == '\0')) {
1720				trace_probe_log_err(0, BAD_VAR);
1721				ret = -ENOENT;
1722				goto error;
1723			}
1724			/* Note: $argN starts from $arg1 */
1725			ret = sprint_nth_btf_arg(n - 1, type, buf + used,
1726						 bufsize - used, ctx);
1727			if (ret < 0)
1728				goto error;
1729			new_argv[j++] = buf + used;
1730			used += ret + 1;
1731		} else
1732			new_argv[j++] = argv[i];
1733	}
1734
1735	return new_argv;
1736
1737error:
1738	kfree(new_argv);
1739	return ERR_PTR(ret);
1740}
1741
1742void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)
1743{
1744	clear_btf_context(ctx);
1745}
1746
1747int traceprobe_update_arg(struct probe_arg *arg)
1748{
1749	struct fetch_insn *code = arg->code;
1750	long offset;
1751	char *tmp;
1752	char c;
1753	int ret = 0;
1754
1755	while (code && code->op != FETCH_OP_END) {
1756		if (code->op == FETCH_NOP_SYMBOL) {
1757			if (code[1].op != FETCH_OP_IMM)
1758				return -EINVAL;
1759
1760			tmp = strpbrk(code->data, "+-");
1761			if (tmp)
1762				c = *tmp;
1763			ret = traceprobe_split_symbol_offset(code->data,
1764							     &offset);
1765			if (ret)
1766				return ret;
1767
1768			code[1].immediate =
1769				(unsigned long)kallsyms_lookup_name(code->data);
1770			if (tmp)
1771				*tmp = c;
1772			if (!code[1].immediate)
1773				return -ENOENT;
1774			code[1].immediate += offset;
1775		}
1776		code++;
1777	}
1778	return 0;
1779}
1780
1781/* When len=0, we just calculate the needed length */
1782#define LEN_OR_ZERO (len ? len - pos : 0)
1783static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
1784			   enum probe_print_type ptype)
1785{
1786	struct probe_arg *parg;
1787	int i, j;
1788	int pos = 0;
1789	const char *fmt, *arg;
1790
1791	switch (ptype) {
1792	case PROBE_PRINT_NORMAL:
1793		fmt = "(%lx)";
1794		arg = ", REC->" FIELD_STRING_IP;
1795		break;
1796	case PROBE_PRINT_RETURN:
1797		fmt = "(%lx <- %lx)";
1798		arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1799		break;
1800	case PROBE_PRINT_EVENT:
1801		fmt = "";
1802		arg = "";
1803		break;
1804	default:
1805		WARN_ON_ONCE(1);
1806		return 0;
1807	}
1808
1809	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
1810
1811	for (i = 0; i < tp->nr_args; i++) {
1812		parg = tp->args + i;
1813		pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
1814		if (parg->count) {
1815			pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
1816					parg->type->fmt);
1817			for (j = 1; j < parg->count; j++)
1818				pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
1819						parg->type->fmt);
1820			pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
1821		} else
1822			pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
1823					parg->type->fmt);
1824	}
1825
1826	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg);
1827
1828	for (i = 0; i < tp->nr_args; i++) {
1829		parg = tp->args + i;
1830		if (parg->count) {
1831			if (parg->type->is_string)
1832				fmt = ", __get_str(%s[%d])";
1833			else
1834				fmt = ", REC->%s[%d]";
1835			for (j = 0; j < parg->count; j++)
1836				pos += snprintf(buf + pos, LEN_OR_ZERO,
1837						fmt, parg->name, j);
1838		} else {
1839			if (parg->type->is_string)
1840				fmt = ", __get_str(%s)";
1841			else
1842				fmt = ", REC->%s";
1843			pos += snprintf(buf + pos, LEN_OR_ZERO,
1844					fmt, parg->name);
1845		}
1846	}
1847
1848	/* return the length of print_fmt */
1849	return pos;
1850}
1851#undef LEN_OR_ZERO
1852
1853int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
1854{
1855	struct trace_event_call *call = trace_probe_event_call(tp);
1856	int len;
1857	char *print_fmt;
1858
1859	/* First: called with 0 length to calculate the needed length */
1860	len = __set_print_fmt(tp, NULL, 0, ptype);
1861	print_fmt = kmalloc(len + 1, GFP_KERNEL);
1862	if (!print_fmt)
1863		return -ENOMEM;
1864
1865	/* Second: actually write the @print_fmt */
1866	__set_print_fmt(tp, print_fmt, len + 1, ptype);
1867	call->print_fmt = print_fmt;
1868
1869	return 0;
1870}
1871
1872int traceprobe_define_arg_fields(struct trace_event_call *event_call,
1873				 size_t offset, struct trace_probe *tp)
1874{
1875	int ret, i;
1876
1877	/* Set argument names as fields */
1878	for (i = 0; i < tp->nr_args; i++) {
1879		struct probe_arg *parg = &tp->args[i];
1880		const char *fmt = parg->type->fmttype;
1881		int size = parg->type->size;
1882
1883		if (parg->fmt)
1884			fmt = parg->fmt;
1885		if (parg->count)
1886			size *= parg->count;
1887		ret = trace_define_field(event_call, fmt, parg->name,
1888					 offset + parg->offset, size,
1889					 parg->type->is_signed,
1890					 FILTER_OTHER);
1891		if (ret)
1892			return ret;
1893	}
1894	return 0;
1895}
1896
1897static void trace_probe_event_free(struct trace_probe_event *tpe)
1898{
1899	kfree(tpe->class.system);
1900	kfree(tpe->call.name);
1901	kfree(tpe->call.print_fmt);
1902	kfree(tpe);
1903}
1904
1905int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
1906{
1907	if (trace_probe_has_sibling(tp))
1908		return -EBUSY;
1909
1910	list_del_init(&tp->list);
1911	trace_probe_event_free(tp->event);
1912
1913	tp->event = to->event;
1914	list_add_tail(&tp->list, trace_probe_probe_list(to));
1915
1916	return 0;
1917}
1918
1919void trace_probe_unlink(struct trace_probe *tp)
1920{
1921	list_del_init(&tp->list);
1922	if (list_empty(trace_probe_probe_list(tp)))
1923		trace_probe_event_free(tp->event);
1924	tp->event = NULL;
1925}
1926
1927void trace_probe_cleanup(struct trace_probe *tp)
1928{
1929	int i;
1930
1931	for (i = 0; i < tp->nr_args; i++)
1932		traceprobe_free_probe_arg(&tp->args[i]);
1933
1934	if (tp->entry_arg) {
1935		kfree(tp->entry_arg->code);
1936		kfree(tp->entry_arg);
1937		tp->entry_arg = NULL;
1938	}
1939
1940	if (tp->event)
1941		trace_probe_unlink(tp);
1942}
1943
1944int trace_probe_init(struct trace_probe *tp, const char *event,
1945		     const char *group, bool alloc_filter, int nargs)
1946{
1947	struct trace_event_call *call;
1948	size_t size = sizeof(struct trace_probe_event);
1949	int ret = 0;
1950
1951	if (!event || !group)
1952		return -EINVAL;
1953
1954	if (alloc_filter)
1955		size += sizeof(struct trace_uprobe_filter);
1956
1957	tp->event = kzalloc(size, GFP_KERNEL);
1958	if (!tp->event)
1959		return -ENOMEM;
1960
1961	INIT_LIST_HEAD(&tp->event->files);
1962	INIT_LIST_HEAD(&tp->event->class.fields);
1963	INIT_LIST_HEAD(&tp->event->probes);
1964	INIT_LIST_HEAD(&tp->list);
1965	list_add(&tp->list, &tp->event->probes);
1966
1967	call = trace_probe_event_call(tp);
1968	call->class = &tp->event->class;
1969	call->name = kstrdup(event, GFP_KERNEL);
1970	if (!call->name) {
1971		ret = -ENOMEM;
1972		goto error;
1973	}
1974
1975	tp->event->class.system = kstrdup(group, GFP_KERNEL);
1976	if (!tp->event->class.system) {
1977		ret = -ENOMEM;
1978		goto error;
1979	}
1980
1981	tp->nr_args = nargs;
1982	/* Make sure pointers in args[] are NULL */
1983	if (nargs)
1984		memset(tp->args, 0, sizeof(tp->args[0]) * nargs);
1985
1986	return 0;
1987
1988error:
1989	trace_probe_cleanup(tp);
1990	return ret;
1991}
1992
1993static struct trace_event_call *
1994find_trace_event_call(const char *system, const char *event_name)
1995{
1996	struct trace_event_call *tp_event;
1997	const char *name;
1998
1999	list_for_each_entry(tp_event, &ftrace_events, list) {
2000		if (!tp_event->class->system ||
2001		    strcmp(system, tp_event->class->system))
2002			continue;
2003		name = trace_event_name(tp_event);
2004		if (!name || strcmp(event_name, name))
2005			continue;
2006		return tp_event;
2007	}
2008
2009	return NULL;
2010}
2011
2012int trace_probe_register_event_call(struct trace_probe *tp)
2013{
2014	struct trace_event_call *call = trace_probe_event_call(tp);
2015	int ret;
2016
2017	lockdep_assert_held(&event_mutex);
2018
2019	if (find_trace_event_call(trace_probe_group_name(tp),
2020				  trace_probe_name(tp)))
2021		return -EEXIST;
2022
2023	ret = register_trace_event(&call->event);
2024	if (!ret)
2025		return -ENODEV;
2026
2027	ret = trace_add_event_call(call);
2028	if (ret)
2029		unregister_trace_event(&call->event);
2030
2031	return ret;
2032}
2033
2034int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
2035{
2036	struct event_file_link *link;
2037
2038	link = kmalloc(sizeof(*link), GFP_KERNEL);
2039	if (!link)
2040		return -ENOMEM;
2041
2042	link->file = file;
2043	INIT_LIST_HEAD(&link->list);
2044	list_add_tail_rcu(&link->list, &tp->event->files);
2045	trace_probe_set_flag(tp, TP_FLAG_TRACE);
2046	return 0;
2047}
2048
2049struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
2050						  struct trace_event_file *file)
2051{
2052	struct event_file_link *link;
2053
2054	trace_probe_for_each_link(link, tp) {
2055		if (link->file == file)
2056			return link;
2057	}
2058
2059	return NULL;
2060}
2061
2062int trace_probe_remove_file(struct trace_probe *tp,
2063			    struct trace_event_file *file)
2064{
2065	struct event_file_link *link;
2066
2067	link = trace_probe_get_file_link(tp, file);
2068	if (!link)
2069		return -ENOENT;
2070
2071	list_del_rcu(&link->list);
2072	kvfree_rcu_mightsleep(link);
2073
2074	if (list_empty(&tp->event->files))
2075		trace_probe_clear_flag(tp, TP_FLAG_TRACE);
2076
2077	return 0;
2078}
2079
2080/*
2081 * Return the smallest index of different type argument (start from 1).
2082 * If all argument types and name are same, return 0.
2083 */
2084int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
2085{
2086	int i;
2087
2088	/* In case of more arguments */
2089	if (a->nr_args < b->nr_args)
2090		return a->nr_args + 1;
2091	if (a->nr_args > b->nr_args)
2092		return b->nr_args + 1;
2093
2094	for (i = 0; i < a->nr_args; i++) {
2095		if ((b->nr_args <= i) ||
2096		    ((a->args[i].type != b->args[i].type) ||
2097		     (a->args[i].count != b->args[i].count) ||
2098		     strcmp(a->args[i].name, b->args[i].name)))
2099			return i + 1;
2100	}
2101
2102	return 0;
2103}
2104
2105bool trace_probe_match_command_args(struct trace_probe *tp,
2106				    int argc, const char **argv)
2107{
2108	char buf[MAX_ARGSTR_LEN + 1];
2109	int i;
2110
2111	if (tp->nr_args < argc)
2112		return false;
2113
2114	for (i = 0; i < argc; i++) {
2115		snprintf(buf, sizeof(buf), "%s=%s",
2116			 tp->args[i].name, tp->args[i].comm);
2117		if (strcmp(buf, argv[i]))
2118			return false;
2119	}
2120	return true;
2121}
2122
2123int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
2124{
2125	int argc = 0, ret = 0;
2126	char **argv;
2127
2128	argv = argv_split(GFP_KERNEL, raw_command, &argc);
2129	if (!argv)
2130		return -ENOMEM;
2131
2132	if (argc)
2133		ret = createfn(argc, (const char **)argv);
2134
2135	argv_free(argv);
2136
2137	return ret;
2138}
2139
2140int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
2141		 u8 *data, void *field)
2142{
2143	void *p;
2144	int i, j;
2145
2146	for (i = 0; i < nr_args; i++) {
2147		struct probe_arg *a = args + i;
2148
2149		trace_seq_printf(s, " %s=", a->name);
2150		if (likely(!a->count)) {
2151			if (!a->type->print(s, data + a->offset, field))
2152				return -ENOMEM;
2153			continue;
2154		}
2155		trace_seq_putc(s, '{');
2156		p = data + a->offset;
2157		for (j = 0; j < a->count; j++) {
2158			if (!a->type->print(s, p, field))
2159				return -ENOMEM;
2160			trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
2161			p += a->type->size;
2162		}
2163	}
2164	return 0;
2165}
2166