1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2002 Richard Henderson
4 * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
6 * Copyright (C) 2024 Mike Rapoport IBM.
7 */
8
9#include <linux/mm.h>
10#include <linux/vmalloc.h>
11#include <linux/execmem.h>
12#include <linux/moduleloader.h>
13
14static struct execmem_info *execmem_info __ro_after_init;
15static struct execmem_info default_execmem_info __ro_after_init;
16
17static void *__execmem_alloc(struct execmem_range *range, size_t size)
18{
19	bool kasan = range->flags & EXECMEM_KASAN_SHADOW;
20	unsigned long vm_flags  = VM_FLUSH_RESET_PERMS;
21	gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN;
22	unsigned long start = range->start;
23	unsigned long end = range->end;
24	unsigned int align = range->alignment;
25	pgprot_t pgprot = range->pgprot;
26	void *p;
27
28	if (kasan)
29		vm_flags |= VM_DEFER_KMEMLEAK;
30
31	p = __vmalloc_node_range(size, align, start, end, gfp_flags,
32				 pgprot, vm_flags, NUMA_NO_NODE,
33				 __builtin_return_address(0));
34	if (!p && range->fallback_start) {
35		start = range->fallback_start;
36		end = range->fallback_end;
37		p = __vmalloc_node_range(size, align, start, end, gfp_flags,
38					 pgprot, vm_flags, NUMA_NO_NODE,
39					 __builtin_return_address(0));
40	}
41
42	if (!p) {
43		pr_warn_ratelimited("execmem: unable to allocate memory\n");
44		return NULL;
45	}
46
47	if (kasan && (kasan_alloc_module_shadow(p, size, GFP_KERNEL) < 0)) {
48		vfree(p);
49		return NULL;
50	}
51
52	return kasan_reset_tag(p);
53}
54
55void *execmem_alloc(enum execmem_type type, size_t size)
56{
57	struct execmem_range *range = &execmem_info->ranges[type];
58
59	return __execmem_alloc(range, size);
60}
61
62void execmem_free(void *ptr)
63{
64	/*
65	 * This memory may be RO, and freeing RO memory in an interrupt is not
66	 * supported by vmalloc.
67	 */
68	WARN_ON(in_interrupt());
69	vfree(ptr);
70}
71
72static bool execmem_validate(struct execmem_info *info)
73{
74	struct execmem_range *r = &info->ranges[EXECMEM_DEFAULT];
75
76	if (!r->alignment || !r->start || !r->end || !pgprot_val(r->pgprot)) {
77		pr_crit("Invalid parameters for execmem allocator, module loading will fail");
78		return false;
79	}
80
81	return true;
82}
83
84static void execmem_init_missing(struct execmem_info *info)
85{
86	struct execmem_range *default_range = &info->ranges[EXECMEM_DEFAULT];
87
88	for (int i = EXECMEM_DEFAULT + 1; i < EXECMEM_TYPE_MAX; i++) {
89		struct execmem_range *r = &info->ranges[i];
90
91		if (!r->start) {
92			if (i == EXECMEM_MODULE_DATA)
93				r->pgprot = PAGE_KERNEL;
94			else
95				r->pgprot = default_range->pgprot;
96			r->alignment = default_range->alignment;
97			r->start = default_range->start;
98			r->end = default_range->end;
99			r->flags = default_range->flags;
100			r->fallback_start = default_range->fallback_start;
101			r->fallback_end = default_range->fallback_end;
102		}
103	}
104}
105
106struct execmem_info * __weak execmem_arch_setup(void)
107{
108	return NULL;
109}
110
111static void __init __execmem_init(void)
112{
113	struct execmem_info *info = execmem_arch_setup();
114
115	if (!info) {
116		info = execmem_info = &default_execmem_info;
117		info->ranges[EXECMEM_DEFAULT].start = VMALLOC_START;
118		info->ranges[EXECMEM_DEFAULT].end = VMALLOC_END;
119		info->ranges[EXECMEM_DEFAULT].pgprot = PAGE_KERNEL_EXEC;
120		info->ranges[EXECMEM_DEFAULT].alignment = 1;
121	}
122
123	if (!execmem_validate(info))
124		return;
125
126	execmem_init_missing(info);
127
128	execmem_info = info;
129}
130
131#ifdef CONFIG_ARCH_WANTS_EXECMEM_LATE
132static int __init execmem_late_init(void)
133{
134	__execmem_init();
135	return 0;
136}
137core_initcall(execmem_late_init);
138#else
139void __init execmem_init(void)
140{
141	__execmem_init();
142}
143#endif
144