1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/* Module internals
3 *
4 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
7 */
8
9#include <linux/elf.h>
10#include <linux/compiler.h>
11#include <linux/module.h>
12#include <linux/mutex.h>
13#include <linux/rculist.h>
14#include <linux/rcupdate.h>
15#include <linux/mm.h>
16
17#ifndef ARCH_SHF_SMALL
18#define ARCH_SHF_SMALL 0
19#endif
20
21/*
22 * Use highest 4 bits of sh_entsize to store the mod_mem_type of this
23 * section. This leaves 28 bits for offset on 32-bit systems, which is
24 * about 256 MiB (WARN_ON_ONCE if we exceed that).
25 */
26
27#define SH_ENTSIZE_TYPE_BITS	4
28#define SH_ENTSIZE_TYPE_SHIFT	(BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)
29#define SH_ENTSIZE_TYPE_MASK	((1UL << SH_ENTSIZE_TYPE_BITS) - 1)
30#define SH_ENTSIZE_OFFSET_MASK	((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1)
31
32/* Maximum number of characters written by module_flags() */
33#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
34
35struct kernel_symbol {
36#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
37	int value_offset;
38	int name_offset;
39	int namespace_offset;
40#else
41	unsigned long value;
42	const char *name;
43	const char *namespace;
44#endif
45};
46
47extern struct mutex module_mutex;
48extern struct list_head modules;
49
50extern struct module_attribute *modinfo_attrs[];
51extern size_t modinfo_attrs_count;
52
53/* Provided by the linker */
54extern const struct kernel_symbol __start___ksymtab[];
55extern const struct kernel_symbol __stop___ksymtab[];
56extern const struct kernel_symbol __start___ksymtab_gpl[];
57extern const struct kernel_symbol __stop___ksymtab_gpl[];
58extern const s32 __start___kcrctab[];
59extern const s32 __start___kcrctab_gpl[];
60
61struct load_info {
62	const char *name;
63	/* pointer to module in temporary copy, freed at end of load_module() */
64	struct module *mod;
65	Elf_Ehdr *hdr;
66	unsigned long len;
67	Elf_Shdr *sechdrs;
68	char *secstrings, *strtab;
69	unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs;
70	bool sig_ok;
71#ifdef CONFIG_KALLSYMS
72	unsigned long mod_kallsyms_init_off;
73#endif
74#ifdef CONFIG_MODULE_DECOMPRESS
75#ifdef CONFIG_MODULE_STATS
76	unsigned long compressed_len;
77#endif
78	struct page **pages;
79	unsigned int max_pages;
80	unsigned int used_pages;
81#endif
82	struct {
83		unsigned int sym, str, mod, vers, info, pcpu;
84	} index;
85};
86
87enum mod_license {
88	NOT_GPL_ONLY,
89	GPL_ONLY,
90};
91
92struct find_symbol_arg {
93	/* Input */
94	const char *name;
95	bool gplok;
96	bool warn;
97
98	/* Output */
99	struct module *owner;
100	const s32 *crc;
101	const struct kernel_symbol *sym;
102	enum mod_license license;
103};
104
105int mod_verify_sig(const void *mod, struct load_info *info);
106int try_to_force_load(struct module *mod, const char *reason);
107bool find_symbol(struct find_symbol_arg *fsa);
108struct module *find_module_all(const char *name, size_t len, bool even_unformed);
109int cmp_name(const void *name, const void *sym);
110long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
111				Elf_Shdr *sechdr, unsigned int section);
112char *module_flags(struct module *mod, char *buf, bool show_state);
113size_t module_flags_taint(unsigned long taints, char *buf);
114
115char *module_next_tag_pair(char *string, unsigned long *secsize);
116
117#define for_each_modinfo_entry(entry, info, name) \
118	for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry))
119
120static inline void module_assert_mutex_or_preempt(void)
121{
122#ifdef CONFIG_LOCKDEP
123	if (unlikely(!debug_locks))
124		return;
125
126	WARN_ON_ONCE(!rcu_read_lock_sched_held() &&
127		     !lockdep_is_held(&module_mutex));
128#endif
129}
130
131static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym)
132{
133#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
134	return (unsigned long)offset_to_ptr(&sym->value_offset);
135#else
136	return sym->value;
137#endif
138}
139
140#ifdef CONFIG_LIVEPATCH
141int copy_module_elf(struct module *mod, struct load_info *info);
142void free_module_elf(struct module *mod);
143#else /* !CONFIG_LIVEPATCH */
144static inline int copy_module_elf(struct module *mod, struct load_info *info)
145{
146	return 0;
147}
148
149static inline void free_module_elf(struct module *mod) { }
150#endif /* CONFIG_LIVEPATCH */
151
152static inline bool set_livepatch_module(struct module *mod)
153{
154#ifdef CONFIG_LIVEPATCH
155	mod->klp = true;
156	return true;
157#else
158	return false;
159#endif
160}
161
162/**
163 * enum fail_dup_mod_reason - state at which a duplicate module was detected
164 *
165 * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but
166 * 	we've determined that another module with the same name is already loaded
167 * 	or being processed on our &modules list. This happens on early_mod_check()
168 * 	right before layout_and_allocate(). The kernel would have already
169 * 	vmalloc()'d space for the entire module through finit_module(). If
170 * 	decompression was used two vmap() spaces were used. These failures can
171 * 	happen when userspace has not seen the module present on the kernel and
172 * 	tries to load the module multiple times at same time.
173 * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation
174 *	checks and the kernel determines that the module was unique and because
175 *	of this allocated yet another private kernel copy of the module space in
176 *	layout_and_allocate() but after this determined in add_unformed_module()
177 *	that another module with the same name is already loaded or being processed.
178 *	These failures should be mitigated as much as possible and are indicative
179 *	of really fast races in loading modules. Without module decompression
180 *	they waste twice as much vmap space. With module decompression three
181 *	times the module's size vmap space is wasted.
182 */
183enum fail_dup_mod_reason {
184	FAIL_DUP_MOD_BECOMING = 0,
185	FAIL_DUP_MOD_LOAD,
186};
187
188#ifdef CONFIG_MODULE_DEBUGFS
189extern struct dentry *mod_debugfs_root;
190#endif
191
192#ifdef CONFIG_MODULE_STATS
193
194#define mod_stat_add_long(count, var) atomic_long_add(count, var)
195#define mod_stat_inc(name) atomic_inc(name)
196
197extern atomic_long_t total_mod_size;
198extern atomic_long_t total_text_size;
199extern atomic_long_t invalid_kread_bytes;
200extern atomic_long_t invalid_decompress_bytes;
201
202extern atomic_t modcount;
203extern atomic_t failed_kreads;
204extern atomic_t failed_decompress;
205struct mod_fail_load {
206	struct list_head list;
207	char name[MODULE_NAME_LEN];
208	atomic_long_t count;
209	unsigned long dup_fail_mask;
210};
211
212int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason);
213void mod_stat_bump_invalid(struct load_info *info, int flags);
214void mod_stat_bump_becoming(struct load_info *info, int flags);
215
216#else
217
218#define mod_stat_add_long(name, var)
219#define mod_stat_inc(name)
220
221static inline int try_add_failed_module(const char *name,
222					enum fail_dup_mod_reason reason)
223{
224	return 0;
225}
226
227static inline void mod_stat_bump_invalid(struct load_info *info, int flags)
228{
229}
230
231static inline void mod_stat_bump_becoming(struct load_info *info, int flags)
232{
233}
234
235#endif /* CONFIG_MODULE_STATS */
236
237#ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS
238bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret);
239void kmod_dup_request_announce(char *module_name, int ret);
240#else
241static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret)
242{
243	return false;
244}
245
246static inline void kmod_dup_request_announce(char *module_name, int ret)
247{
248}
249#endif
250
251#ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING
252struct mod_unload_taint {
253	struct list_head list;
254	char name[MODULE_NAME_LEN];
255	unsigned long taints;
256	u64 count;
257};
258
259int try_add_tainted_module(struct module *mod);
260void print_unloaded_tainted_modules(void);
261#else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
262static inline int try_add_tainted_module(struct module *mod)
263{
264	return 0;
265}
266
267static inline void print_unloaded_tainted_modules(void)
268{
269}
270#endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
271
272#ifdef CONFIG_MODULE_DECOMPRESS
273int module_decompress(struct load_info *info, const void *buf, size_t size);
274void module_decompress_cleanup(struct load_info *info);
275#else
276static inline int module_decompress(struct load_info *info,
277				    const void *buf, size_t size)
278{
279	return -EOPNOTSUPP;
280}
281
282static inline void module_decompress_cleanup(struct load_info *info)
283{
284}
285#endif
286
287struct mod_tree_root {
288#ifdef CONFIG_MODULES_TREE_LOOKUP
289	struct latch_tree_root root;
290#endif
291	unsigned long addr_min;
292	unsigned long addr_max;
293#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
294	unsigned long data_addr_min;
295	unsigned long data_addr_max;
296#endif
297};
298
299extern struct mod_tree_root mod_tree;
300
301#ifdef CONFIG_MODULES_TREE_LOOKUP
302void mod_tree_insert(struct module *mod);
303void mod_tree_remove_init(struct module *mod);
304void mod_tree_remove(struct module *mod);
305struct module *mod_find(unsigned long addr, struct mod_tree_root *tree);
306#else /* !CONFIG_MODULES_TREE_LOOKUP */
307
308static inline void mod_tree_insert(struct module *mod) { }
309static inline void mod_tree_remove_init(struct module *mod) { }
310static inline void mod_tree_remove(struct module *mod) { }
311static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree)
312{
313	struct module *mod;
314
315	list_for_each_entry_rcu(mod, &modules, list,
316				lockdep_is_held(&module_mutex)) {
317		if (within_module(addr, mod))
318			return mod;
319	}
320
321	return NULL;
322}
323#endif /* CONFIG_MODULES_TREE_LOOKUP */
324
325int module_enable_rodata_ro(const struct module *mod, bool after_init);
326int module_enable_data_nx(const struct module *mod);
327int module_enable_text_rox(const struct module *mod);
328int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
329				char *secstrings, struct module *mod);
330
331#ifdef CONFIG_MODULE_SIG
332int module_sig_check(struct load_info *info, int flags);
333#else /* !CONFIG_MODULE_SIG */
334static inline int module_sig_check(struct load_info *info, int flags)
335{
336	return 0;
337}
338#endif /* !CONFIG_MODULE_SIG */
339
340#ifdef CONFIG_DEBUG_KMEMLEAK
341void kmemleak_load_module(const struct module *mod, const struct load_info *info);
342#else /* !CONFIG_DEBUG_KMEMLEAK */
343static inline void kmemleak_load_module(const struct module *mod,
344					const struct load_info *info) { }
345#endif /* CONFIG_DEBUG_KMEMLEAK */
346
347#ifdef CONFIG_KALLSYMS
348void init_build_id(struct module *mod, const struct load_info *info);
349void layout_symtab(struct module *mod, struct load_info *info);
350void add_kallsyms(struct module *mod, const struct load_info *info);
351
352static inline bool sect_empty(const Elf_Shdr *sect)
353{
354	return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
355}
356#else /* !CONFIG_KALLSYMS */
357static inline void init_build_id(struct module *mod, const struct load_info *info) { }
358static inline void layout_symtab(struct module *mod, struct load_info *info) { }
359static inline void add_kallsyms(struct module *mod, const struct load_info *info) { }
360#endif /* CONFIG_KALLSYMS */
361
362#ifdef CONFIG_SYSFS
363int mod_sysfs_setup(struct module *mod, const struct load_info *info,
364		    struct kernel_param *kparam, unsigned int num_params);
365void mod_sysfs_teardown(struct module *mod);
366void init_param_lock(struct module *mod);
367#else /* !CONFIG_SYSFS */
368static inline int mod_sysfs_setup(struct module *mod,
369			   	  const struct load_info *info,
370			   	  struct kernel_param *kparam,
371			   	  unsigned int num_params)
372{
373	return 0;
374}
375
376static inline void mod_sysfs_teardown(struct module *mod) { }
377static inline void init_param_lock(struct module *mod) { }
378#endif /* CONFIG_SYSFS */
379
380#ifdef CONFIG_MODVERSIONS
381int check_version(const struct load_info *info,
382		  const char *symname, struct module *mod, const s32 *crc);
383void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp,
384		   struct kernel_symbol *ks, struct tracepoint * const *tp);
385int check_modstruct_version(const struct load_info *info, struct module *mod);
386int same_magic(const char *amagic, const char *bmagic, bool has_crcs);
387#else /* !CONFIG_MODVERSIONS */
388static inline int check_version(const struct load_info *info,
389				const char *symname,
390				struct module *mod,
391				const s32 *crc)
392{
393	return 1;
394}
395
396static inline int check_modstruct_version(const struct load_info *info,
397					  struct module *mod)
398{
399	return 1;
400}
401
402static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs)
403{
404	return strcmp(amagic, bmagic) == 0;
405}
406#endif /* CONFIG_MODVERSIONS */
407