1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* Module internals 3 * 4 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org> 7 */ 8 9#include <linux/elf.h> 10#include <linux/compiler.h> 11#include <linux/module.h> 12#include <linux/mutex.h> 13#include <linux/rculist.h> 14#include <linux/rcupdate.h> 15#include <linux/mm.h> 16 17#ifndef ARCH_SHF_SMALL 18#define ARCH_SHF_SMALL 0 19#endif 20 21/* 22 * Use highest 4 bits of sh_entsize to store the mod_mem_type of this 23 * section. This leaves 28 bits for offset on 32-bit systems, which is 24 * about 256 MiB (WARN_ON_ONCE if we exceed that). 25 */ 26 27#define SH_ENTSIZE_TYPE_BITS 4 28#define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS) 29#define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1) 30#define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1) 31 32/* Maximum number of characters written by module_flags() */ 33#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4) 34 35struct kernel_symbol { 36#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS 37 int value_offset; 38 int name_offset; 39 int namespace_offset; 40#else 41 unsigned long value; 42 const char *name; 43 const char *namespace; 44#endif 45}; 46 47extern struct mutex module_mutex; 48extern struct list_head modules; 49 50extern struct module_attribute *modinfo_attrs[]; 51extern size_t modinfo_attrs_count; 52 53/* Provided by the linker */ 54extern const struct kernel_symbol __start___ksymtab[]; 55extern const struct kernel_symbol __stop___ksymtab[]; 56extern const struct kernel_symbol __start___ksymtab_gpl[]; 57extern const struct kernel_symbol __stop___ksymtab_gpl[]; 58extern const s32 __start___kcrctab[]; 59extern const s32 __start___kcrctab_gpl[]; 60 61struct load_info { 62 const char *name; 63 /* pointer to module in temporary copy, freed at end of load_module() */ 64 struct module *mod; 65 Elf_Ehdr *hdr; 66 unsigned long len; 67 Elf_Shdr *sechdrs; 68 char *secstrings, *strtab; 69 unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs; 70 bool sig_ok; 71#ifdef CONFIG_KALLSYMS 72 unsigned long mod_kallsyms_init_off; 73#endif 74#ifdef CONFIG_MODULE_DECOMPRESS 75#ifdef CONFIG_MODULE_STATS 76 unsigned long compressed_len; 77#endif 78 struct page **pages; 79 unsigned int max_pages; 80 unsigned int used_pages; 81#endif 82 struct { 83 unsigned int sym, str, mod, vers, info, pcpu; 84 } index; 85}; 86 87enum mod_license { 88 NOT_GPL_ONLY, 89 GPL_ONLY, 90}; 91 92struct find_symbol_arg { 93 /* Input */ 94 const char *name; 95 bool gplok; 96 bool warn; 97 98 /* Output */ 99 struct module *owner; 100 const s32 *crc; 101 const struct kernel_symbol *sym; 102 enum mod_license license; 103}; 104 105int mod_verify_sig(const void *mod, struct load_info *info); 106int try_to_force_load(struct module *mod, const char *reason); 107bool find_symbol(struct find_symbol_arg *fsa); 108struct module *find_module_all(const char *name, size_t len, bool even_unformed); 109int cmp_name(const void *name, const void *sym); 110long module_get_offset_and_type(struct module *mod, enum mod_mem_type type, 111 Elf_Shdr *sechdr, unsigned int section); 112char *module_flags(struct module *mod, char *buf, bool show_state); 113size_t module_flags_taint(unsigned long taints, char *buf); 114 115char *module_next_tag_pair(char *string, unsigned long *secsize); 116 117#define for_each_modinfo_entry(entry, info, name) \ 118 for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry)) 119 120static inline void module_assert_mutex_or_preempt(void) 121{ 122#ifdef CONFIG_LOCKDEP 123 if (unlikely(!debug_locks)) 124 return; 125 126 WARN_ON_ONCE(!rcu_read_lock_sched_held() && 127 !lockdep_is_held(&module_mutex)); 128#endif 129} 130 131static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym) 132{ 133#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS 134 return (unsigned long)offset_to_ptr(&sym->value_offset); 135#else 136 return sym->value; 137#endif 138} 139 140#ifdef CONFIG_LIVEPATCH 141int copy_module_elf(struct module *mod, struct load_info *info); 142void free_module_elf(struct module *mod); 143#else /* !CONFIG_LIVEPATCH */ 144static inline int copy_module_elf(struct module *mod, struct load_info *info) 145{ 146 return 0; 147} 148 149static inline void free_module_elf(struct module *mod) { } 150#endif /* CONFIG_LIVEPATCH */ 151 152static inline bool set_livepatch_module(struct module *mod) 153{ 154#ifdef CONFIG_LIVEPATCH 155 mod->klp = true; 156 return true; 157#else 158 return false; 159#endif 160} 161 162/** 163 * enum fail_dup_mod_reason - state at which a duplicate module was detected 164 * 165 * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but 166 * we've determined that another module with the same name is already loaded 167 * or being processed on our &modules list. This happens on early_mod_check() 168 * right before layout_and_allocate(). The kernel would have already 169 * vmalloc()'d space for the entire module through finit_module(). If 170 * decompression was used two vmap() spaces were used. These failures can 171 * happen when userspace has not seen the module present on the kernel and 172 * tries to load the module multiple times at same time. 173 * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation 174 * checks and the kernel determines that the module was unique and because 175 * of this allocated yet another private kernel copy of the module space in 176 * layout_and_allocate() but after this determined in add_unformed_module() 177 * that another module with the same name is already loaded or being processed. 178 * These failures should be mitigated as much as possible and are indicative 179 * of really fast races in loading modules. Without module decompression 180 * they waste twice as much vmap space. With module decompression three 181 * times the module's size vmap space is wasted. 182 */ 183enum fail_dup_mod_reason { 184 FAIL_DUP_MOD_BECOMING = 0, 185 FAIL_DUP_MOD_LOAD, 186}; 187 188#ifdef CONFIG_MODULE_DEBUGFS 189extern struct dentry *mod_debugfs_root; 190#endif 191 192#ifdef CONFIG_MODULE_STATS 193 194#define mod_stat_add_long(count, var) atomic_long_add(count, var) 195#define mod_stat_inc(name) atomic_inc(name) 196 197extern atomic_long_t total_mod_size; 198extern atomic_long_t total_text_size; 199extern atomic_long_t invalid_kread_bytes; 200extern atomic_long_t invalid_decompress_bytes; 201 202extern atomic_t modcount; 203extern atomic_t failed_kreads; 204extern atomic_t failed_decompress; 205struct mod_fail_load { 206 struct list_head list; 207 char name[MODULE_NAME_LEN]; 208 atomic_long_t count; 209 unsigned long dup_fail_mask; 210}; 211 212int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason); 213void mod_stat_bump_invalid(struct load_info *info, int flags); 214void mod_stat_bump_becoming(struct load_info *info, int flags); 215 216#else 217 218#define mod_stat_add_long(name, var) 219#define mod_stat_inc(name) 220 221static inline int try_add_failed_module(const char *name, 222 enum fail_dup_mod_reason reason) 223{ 224 return 0; 225} 226 227static inline void mod_stat_bump_invalid(struct load_info *info, int flags) 228{ 229} 230 231static inline void mod_stat_bump_becoming(struct load_info *info, int flags) 232{ 233} 234 235#endif /* CONFIG_MODULE_STATS */ 236 237#ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS 238bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret); 239void kmod_dup_request_announce(char *module_name, int ret); 240#else 241static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) 242{ 243 return false; 244} 245 246static inline void kmod_dup_request_announce(char *module_name, int ret) 247{ 248} 249#endif 250 251#ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING 252struct mod_unload_taint { 253 struct list_head list; 254 char name[MODULE_NAME_LEN]; 255 unsigned long taints; 256 u64 count; 257}; 258 259int try_add_tainted_module(struct module *mod); 260void print_unloaded_tainted_modules(void); 261#else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */ 262static inline int try_add_tainted_module(struct module *mod) 263{ 264 return 0; 265} 266 267static inline void print_unloaded_tainted_modules(void) 268{ 269} 270#endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */ 271 272#ifdef CONFIG_MODULE_DECOMPRESS 273int module_decompress(struct load_info *info, const void *buf, size_t size); 274void module_decompress_cleanup(struct load_info *info); 275#else 276static inline int module_decompress(struct load_info *info, 277 const void *buf, size_t size) 278{ 279 return -EOPNOTSUPP; 280} 281 282static inline void module_decompress_cleanup(struct load_info *info) 283{ 284} 285#endif 286 287struct mod_tree_root { 288#ifdef CONFIG_MODULES_TREE_LOOKUP 289 struct latch_tree_root root; 290#endif 291 unsigned long addr_min; 292 unsigned long addr_max; 293#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC 294 unsigned long data_addr_min; 295 unsigned long data_addr_max; 296#endif 297}; 298 299extern struct mod_tree_root mod_tree; 300 301#ifdef CONFIG_MODULES_TREE_LOOKUP 302void mod_tree_insert(struct module *mod); 303void mod_tree_remove_init(struct module *mod); 304void mod_tree_remove(struct module *mod); 305struct module *mod_find(unsigned long addr, struct mod_tree_root *tree); 306#else /* !CONFIG_MODULES_TREE_LOOKUP */ 307 308static inline void mod_tree_insert(struct module *mod) { } 309static inline void mod_tree_remove_init(struct module *mod) { } 310static inline void mod_tree_remove(struct module *mod) { } 311static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree) 312{ 313 struct module *mod; 314 315 list_for_each_entry_rcu(mod, &modules, list, 316 lockdep_is_held(&module_mutex)) { 317 if (within_module(addr, mod)) 318 return mod; 319 } 320 321 return NULL; 322} 323#endif /* CONFIG_MODULES_TREE_LOOKUP */ 324 325int module_enable_rodata_ro(const struct module *mod, bool after_init); 326int module_enable_data_nx(const struct module *mod); 327int module_enable_text_rox(const struct module *mod); 328int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, 329 char *secstrings, struct module *mod); 330 331#ifdef CONFIG_MODULE_SIG 332int module_sig_check(struct load_info *info, int flags); 333#else /* !CONFIG_MODULE_SIG */ 334static inline int module_sig_check(struct load_info *info, int flags) 335{ 336 return 0; 337} 338#endif /* !CONFIG_MODULE_SIG */ 339 340#ifdef CONFIG_DEBUG_KMEMLEAK 341void kmemleak_load_module(const struct module *mod, const struct load_info *info); 342#else /* !CONFIG_DEBUG_KMEMLEAK */ 343static inline void kmemleak_load_module(const struct module *mod, 344 const struct load_info *info) { } 345#endif /* CONFIG_DEBUG_KMEMLEAK */ 346 347#ifdef CONFIG_KALLSYMS 348void init_build_id(struct module *mod, const struct load_info *info); 349void layout_symtab(struct module *mod, struct load_info *info); 350void add_kallsyms(struct module *mod, const struct load_info *info); 351 352static inline bool sect_empty(const Elf_Shdr *sect) 353{ 354 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0; 355} 356#else /* !CONFIG_KALLSYMS */ 357static inline void init_build_id(struct module *mod, const struct load_info *info) { } 358static inline void layout_symtab(struct module *mod, struct load_info *info) { } 359static inline void add_kallsyms(struct module *mod, const struct load_info *info) { } 360#endif /* CONFIG_KALLSYMS */ 361 362#ifdef CONFIG_SYSFS 363int mod_sysfs_setup(struct module *mod, const struct load_info *info, 364 struct kernel_param *kparam, unsigned int num_params); 365void mod_sysfs_teardown(struct module *mod); 366void init_param_lock(struct module *mod); 367#else /* !CONFIG_SYSFS */ 368static inline int mod_sysfs_setup(struct module *mod, 369 const struct load_info *info, 370 struct kernel_param *kparam, 371 unsigned int num_params) 372{ 373 return 0; 374} 375 376static inline void mod_sysfs_teardown(struct module *mod) { } 377static inline void init_param_lock(struct module *mod) { } 378#endif /* CONFIG_SYSFS */ 379 380#ifdef CONFIG_MODVERSIONS 381int check_version(const struct load_info *info, 382 const char *symname, struct module *mod, const s32 *crc); 383void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp, 384 struct kernel_symbol *ks, struct tracepoint * const *tp); 385int check_modstruct_version(const struct load_info *info, struct module *mod); 386int same_magic(const char *amagic, const char *bmagic, bool has_crcs); 387#else /* !CONFIG_MODVERSIONS */ 388static inline int check_version(const struct load_info *info, 389 const char *symname, 390 struct module *mod, 391 const s32 *crc) 392{ 393 return 1; 394} 395 396static inline int check_modstruct_version(const struct load_info *info, 397 struct module *mod) 398{ 399 return 1; 400} 401 402static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs) 403{ 404 return strcmp(amagic, bmagic) == 0; 405} 406#endif /* CONFIG_MODVERSIONS */ 407