1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/alloc_tag.h>
3#include <linux/fs.h>
4#include <linux/gfp.h>
5#include <linux/module.h>
6#include <linux/page_ext.h>
7#include <linux/proc_fs.h>
8#include <linux/seq_buf.h>
9#include <linux/seq_file.h>
10
11static struct codetag_type *alloc_tag_cttype;
12
13DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
14EXPORT_SYMBOL(_shared_alloc_tag);
15
16DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
17			mem_alloc_profiling_key);
18
19struct allocinfo_private {
20	struct codetag_iterator iter;
21	bool print_header;
22};
23
24static void *allocinfo_start(struct seq_file *m, loff_t *pos)
25{
26	struct allocinfo_private *priv;
27	struct codetag *ct;
28	loff_t node = *pos;
29
30	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
31	m->private = priv;
32	if (!priv)
33		return NULL;
34
35	priv->print_header = (node == 0);
36	codetag_lock_module_list(alloc_tag_cttype, true);
37	priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
38	while ((ct = codetag_next_ct(&priv->iter)) != NULL && node)
39		node--;
40
41	return ct ? priv : NULL;
42}
43
44static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
45{
46	struct allocinfo_private *priv = (struct allocinfo_private *)arg;
47	struct codetag *ct = codetag_next_ct(&priv->iter);
48
49	(*pos)++;
50	if (!ct)
51		return NULL;
52
53	return priv;
54}
55
56static void allocinfo_stop(struct seq_file *m, void *arg)
57{
58	struct allocinfo_private *priv = (struct allocinfo_private *)m->private;
59
60	if (priv) {
61		codetag_lock_module_list(alloc_tag_cttype, false);
62		kfree(priv);
63	}
64}
65
66static void print_allocinfo_header(struct seq_buf *buf)
67{
68	/* Output format version, so we can change it. */
69	seq_buf_printf(buf, "allocinfo - version: 1.0\n");
70	seq_buf_printf(buf, "#     <size>  <calls> <tag info>\n");
71}
72
73static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
74{
75	struct alloc_tag *tag = ct_to_alloc_tag(ct);
76	struct alloc_tag_counters counter = alloc_tag_read(tag);
77	s64 bytes = counter.bytes;
78
79	seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
80	codetag_to_text(out, ct);
81	seq_buf_putc(out, ' ');
82	seq_buf_putc(out, '\n');
83}
84
85static int allocinfo_show(struct seq_file *m, void *arg)
86{
87	struct allocinfo_private *priv = (struct allocinfo_private *)arg;
88	char *bufp;
89	size_t n = seq_get_buf(m, &bufp);
90	struct seq_buf buf;
91
92	seq_buf_init(&buf, bufp, n);
93	if (priv->print_header) {
94		print_allocinfo_header(&buf);
95		priv->print_header = false;
96	}
97	alloc_tag_to_text(&buf, priv->iter.ct);
98	seq_commit(m, seq_buf_used(&buf));
99	return 0;
100}
101
102static const struct seq_operations allocinfo_seq_op = {
103	.start	= allocinfo_start,
104	.next	= allocinfo_next,
105	.stop	= allocinfo_stop,
106	.show	= allocinfo_show,
107};
108
109size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
110{
111	struct codetag_iterator iter;
112	struct codetag *ct;
113	struct codetag_bytes n;
114	unsigned int i, nr = 0;
115
116	if (can_sleep)
117		codetag_lock_module_list(alloc_tag_cttype, true);
118	else if (!codetag_trylock_module_list(alloc_tag_cttype))
119		return 0;
120
121	iter = codetag_get_ct_iter(alloc_tag_cttype);
122	while ((ct = codetag_next_ct(&iter))) {
123		struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
124
125		n.ct	= ct;
126		n.bytes = counter.bytes;
127
128		for (i = 0; i < nr; i++)
129			if (n.bytes > tags[i].bytes)
130				break;
131
132		if (i < count) {
133			nr -= nr == count;
134			memmove(&tags[i + 1],
135				&tags[i],
136				sizeof(tags[0]) * (nr - i));
137			nr++;
138			tags[i] = n;
139		}
140	}
141
142	codetag_lock_module_list(alloc_tag_cttype, false);
143
144	return nr;
145}
146
147static void __init procfs_init(void)
148{
149	proc_create_seq("allocinfo", 0400, NULL, &allocinfo_seq_op);
150}
151
152static bool alloc_tag_module_unload(struct codetag_type *cttype,
153				    struct codetag_module *cmod)
154{
155	struct codetag_iterator iter = codetag_get_ct_iter(cttype);
156	struct alloc_tag_counters counter;
157	bool module_unused = true;
158	struct alloc_tag *tag;
159	struct codetag *ct;
160
161	for (ct = codetag_next_ct(&iter); ct; ct = codetag_next_ct(&iter)) {
162		if (iter.cmod != cmod)
163			continue;
164
165		tag = ct_to_alloc_tag(ct);
166		counter = alloc_tag_read(tag);
167
168		if (WARN(counter.bytes,
169			 "%s:%u module %s func:%s has %llu allocated at module unload",
170			 ct->filename, ct->lineno, ct->modname, ct->function, counter.bytes))
171			module_unused = false;
172	}
173
174	return module_unused;
175}
176
177#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
178static bool mem_profiling_support __meminitdata = true;
179#else
180static bool mem_profiling_support __meminitdata;
181#endif
182
183static int __init setup_early_mem_profiling(char *str)
184{
185	bool enable;
186
187	if (!str || !str[0])
188		return -EINVAL;
189
190	if (!strncmp(str, "never", 5)) {
191		enable = false;
192		mem_profiling_support = false;
193	} else {
194		int res;
195
196		res = kstrtobool(str, &enable);
197		if (res)
198			return res;
199
200		mem_profiling_support = true;
201	}
202
203	if (enable != static_key_enabled(&mem_alloc_profiling_key)) {
204		if (enable)
205			static_branch_enable(&mem_alloc_profiling_key);
206		else
207			static_branch_disable(&mem_alloc_profiling_key);
208	}
209
210	return 0;
211}
212early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
213
214static __init bool need_page_alloc_tagging(void)
215{
216	return mem_profiling_support;
217}
218
219static __init void init_page_alloc_tagging(void)
220{
221}
222
223struct page_ext_operations page_alloc_tagging_ops = {
224	.size = sizeof(union codetag_ref),
225	.need = need_page_alloc_tagging,
226	.init = init_page_alloc_tagging,
227};
228EXPORT_SYMBOL(page_alloc_tagging_ops);
229
230static struct ctl_table memory_allocation_profiling_sysctls[] = {
231	{
232		.procname	= "mem_profiling",
233		.data		= &mem_alloc_profiling_key,
234#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
235		.mode		= 0444,
236#else
237		.mode		= 0644,
238#endif
239		.proc_handler	= proc_do_static_key,
240	},
241	{ }
242};
243
244static int __init alloc_tag_init(void)
245{
246	const struct codetag_type_desc desc = {
247		.section	= "alloc_tags",
248		.tag_size	= sizeof(struct alloc_tag),
249		.module_unload	= alloc_tag_module_unload,
250	};
251
252	alloc_tag_cttype = codetag_register_type(&desc);
253	if (IS_ERR(alloc_tag_cttype))
254		return PTR_ERR(alloc_tag_cttype);
255
256	if (!mem_profiling_support)
257		memory_allocation_profiling_sysctls[0].mode = 0444;
258	register_sysctl_init("vm", memory_allocation_profiling_sysctls);
259	procfs_init();
260
261	return 0;
262}
263module_init(alloc_tag_init);
264