1457c8996SThomas Gleixner// SPDX-License-Identifier: GPL-2.0-only
216d69265SAndrew Morton#include <linux/mm.h>
330992c97SMatt Mackall#include <linux/slab.h>
430992c97SMatt Mackall#include <linux/string.h>
53b32123dSGideon Israel Dsouza#include <linux/compiler.h>
6b95f1b31SPaul Gortmaker#include <linux/export.h>
796840aa0SDavi Arnaut#include <linux/err.h>
83b8f14b4SAdrian Bunk#include <linux/sched.h>
96e84f315SIngo Molnar#include <linux/sched/mm.h>
1079eb597cSDaniel Jordan#include <linux/sched/signal.h>
1168db0cf1SIngo Molnar#include <linux/sched/task_stack.h>
12eb36c587SAl Viro#include <linux/security.h>
139800339bSShaohua Li#include <linux/swap.h>
1433806f06SShaohua Li#include <linux/swapops.h>
1500619bccSJerome Marchand#include <linux/mman.h>
1600619bccSJerome Marchand#include <linux/hugetlb.h>
1739f1f78dSAl Viro#include <linux/vmalloc.h>
18897ab3e0SMike Rapoport#include <linux/userfaultfd_k.h>
19649775beSAlexandre Ghiti#include <linux/elf.h>
2067f3977fSAlexandre Ghiti#include <linux/elf-randomize.h>
2167f3977fSAlexandre Ghiti#include <linux/personality.h>
22649775beSAlexandre Ghiti#include <linux/random.h>
2367f3977fSAlexandre Ghiti#include <linux/processor.h>
2467f3977fSAlexandre Ghiti#include <linux/sizes.h>
2567f3977fSAlexandre Ghiti#include <linux/compat.h>
2600619bccSJerome Marchand
277c0f6ba6SLinus Torvalds#include <linux/uaccess.h>
2830992c97SMatt Mackall
296038def0SNamhyung Kim#include "internal.h"
306038def0SNamhyung Kim
31a4bb1e43SAndrzej Hajda/**
32a4bb1e43SAndrzej Hajda * kfree_const - conditionally free memory
33a4bb1e43SAndrzej Hajda * @x: pointer to the memory
34a4bb1e43SAndrzej Hajda *
35a4bb1e43SAndrzej Hajda * Function calls kfree only if @x is not in .rodata section.
36a4bb1e43SAndrzej Hajda */
37a4bb1e43SAndrzej Hajdavoid kfree_const(const void *x)
38a4bb1e43SAndrzej Hajda{
39a4bb1e43SAndrzej Hajda	if (!is_kernel_rodata((unsigned long)x))
40a4bb1e43SAndrzej Hajda		kfree(x);
41a4bb1e43SAndrzej Hajda}
42a4bb1e43SAndrzej HajdaEXPORT_SYMBOL(kfree_const);
43a4bb1e43SAndrzej Hajda
4430992c97SMatt Mackall/**
4530992c97SMatt Mackall * kstrdup - allocate space for and copy an existing string
4630992c97SMatt Mackall * @s: the string to duplicate
4730992c97SMatt Mackall * @gfp: the GFP mask used in the kmalloc() call when allocating memory
48a862f68aSMike Rapoport *
49a862f68aSMike Rapoport * Return: newly allocated copy of @s or %NULL in case of error
5030992c97SMatt Mackall */
5130992c97SMatt Mackallchar *kstrdup(const char *s, gfp_t gfp)
5230992c97SMatt Mackall{
5330992c97SMatt Mackall	size_t len;
5430992c97SMatt Mackall	char *buf;
5530992c97SMatt Mackall
5630992c97SMatt Mackall	if (!s)
5730992c97SMatt Mackall		return NULL;
5830992c97SMatt Mackall
5930992c97SMatt Mackall	len = strlen(s) + 1;
601d2c8eeaSChristoph Hellwig	buf = kmalloc_track_caller(len, gfp);
6130992c97SMatt Mackall	if (buf)
6230992c97SMatt Mackall		memcpy(buf, s, len);
6330992c97SMatt Mackall	return buf;
6430992c97SMatt Mackall}
6530992c97SMatt MackallEXPORT_SYMBOL(kstrdup);
6696840aa0SDavi Arnaut
67a4bb1e43SAndrzej Hajda/**
68a4bb1e43SAndrzej Hajda * kstrdup_const - conditionally duplicate an existing const string
69a4bb1e43SAndrzej Hajda * @s: the string to duplicate
70a4bb1e43SAndrzej Hajda * @gfp: the GFP mask used in the kmalloc() call when allocating memory
71a4bb1e43SAndrzej Hajda *
72295a1730SBartosz Golaszewski * Note: Strings allocated by kstrdup_const should be freed by kfree_const and
73295a1730SBartosz Golaszewski * must not be passed to krealloc().
74a862f68aSMike Rapoport *
75a862f68aSMike Rapoport * Return: source string if it is in .rodata section otherwise
76a862f68aSMike Rapoport * fallback to kstrdup.
77a4bb1e43SAndrzej Hajda */
78a4bb1e43SAndrzej Hajdaconst char *kstrdup_const(const char *s, gfp_t gfp)
79a4bb1e43SAndrzej Hajda{
80a4bb1e43SAndrzej Hajda	if (is_kernel_rodata((unsigned long)s))
81a4bb1e43SAndrzej Hajda		return s;
82a4bb1e43SAndrzej Hajda
83a4bb1e43SAndrzej Hajda	return kstrdup(s, gfp);
84a4bb1e43SAndrzej Hajda}
85a4bb1e43SAndrzej HajdaEXPORT_SYMBOL(kstrdup_const);
86a4bb1e43SAndrzej Hajda
871e66df3eSJeremy Fitzhardinge/**
881e66df3eSJeremy Fitzhardinge * kstrndup - allocate space for and copy an existing string
891e66df3eSJeremy Fitzhardinge * @s: the string to duplicate
901e66df3eSJeremy Fitzhardinge * @max: read at most @max chars from @s
911e66df3eSJeremy Fitzhardinge * @gfp: the GFP mask used in the kmalloc() call when allocating memory
92f3515741SDavid Howells *
93f3515741SDavid Howells * Note: Use kmemdup_nul() instead if the size is known exactly.
94a862f68aSMike Rapoport *
95a862f68aSMike Rapoport * Return: newly allocated copy of @s or %NULL in case of error
961e66df3eSJeremy Fitzhardinge */
971e66df3eSJeremy Fitzhardingechar *kstrndup(const char *s, size_t max, gfp_t gfp)
981e66df3eSJeremy Fitzhardinge{
991e66df3eSJeremy Fitzhardinge	size_t len;
1001e66df3eSJeremy Fitzhardinge	char *buf;
1011e66df3eSJeremy Fitzhardinge
1021e66df3eSJeremy Fitzhardinge	if (!s)
1031e66df3eSJeremy Fitzhardinge		return NULL;
1041e66df3eSJeremy Fitzhardinge
1051e66df3eSJeremy Fitzhardinge	len = strnlen(s, max);
1061e66df3eSJeremy Fitzhardinge	buf = kmalloc_track_caller(len+1, gfp);
1071e66df3eSJeremy Fitzhardinge	if (buf) {
1081e66df3eSJeremy Fitzhardinge		memcpy(buf, s, len);
1091e66df3eSJeremy Fitzhardinge		buf[len] = '\0';
1101e66df3eSJeremy Fitzhardinge	}
1111e66df3eSJeremy Fitzhardinge	return buf;
1121e66df3eSJeremy Fitzhardinge}
1131e66df3eSJeremy FitzhardingeEXPORT_SYMBOL(kstrndup);
1141e66df3eSJeremy Fitzhardinge
1151a2f67b4SAlexey Dobriyan/**
1161a2f67b4SAlexey Dobriyan * kmemdup - duplicate region of memory
1171a2f67b4SAlexey Dobriyan *
1181a2f67b4SAlexey Dobriyan * @src: memory region to duplicate
1191a2f67b4SAlexey Dobriyan * @len: memory region length
1201a2f67b4SAlexey Dobriyan * @gfp: GFP mask to use
121a862f68aSMike Rapoport *
122a862f68aSMike Rapoport * Return: newly allocated copy of @src or %NULL in case of error
1231a2f67b4SAlexey Dobriyan */
1241a2f67b4SAlexey Dobriyanvoid *kmemdup(const void *src, size_t len, gfp_t gfp)
1251a2f67b4SAlexey Dobriyan{
1261a2f67b4SAlexey Dobriyan	void *p;
1271a2f67b4SAlexey Dobriyan
1281d2c8eeaSChristoph Hellwig	p = kmalloc_track_caller(len, gfp);
1291a2f67b4SAlexey Dobriyan	if (p)
1301a2f67b4SAlexey Dobriyan		memcpy(p, src, len);
1311a2f67b4SAlexey Dobriyan	return p;
1321a2f67b4SAlexey Dobriyan}
1331a2f67b4SAlexey DobriyanEXPORT_SYMBOL(kmemdup);
1341a2f67b4SAlexey Dobriyan
135f3515741SDavid Howells/**
136f3515741SDavid Howells * kmemdup_nul - Create a NUL-terminated string from unterminated data
137f3515741SDavid Howells * @s: The data to stringify
138f3515741SDavid Howells * @len: The size of the data
139f3515741SDavid Howells * @gfp: the GFP mask used in the kmalloc() call when allocating memory
140a862f68aSMike Rapoport *
141a862f68aSMike Rapoport * Return: newly allocated copy of @s with NUL-termination or %NULL in
142a862f68aSMike Rapoport * case of error
143f3515741SDavid Howells */
144f3515741SDavid Howellschar *kmemdup_nul(const char *s, size_t len, gfp_t gfp)
145f3515741SDavid Howells{
146f3515741SDavid Howells	char *buf;
147f3515741SDavid Howells
148f3515741SDavid Howells	if (!s)
149f3515741SDavid Howells		return NULL;
150f3515741SDavid Howells
151f3515741SDavid Howells	buf = kmalloc_track_caller(len + 1, gfp);
152f3515741SDavid Howells	if (buf) {
153f3515741SDavid Howells		memcpy(buf, s, len);
154f3515741SDavid Howells		buf[len] = '\0';
155f3515741SDavid Howells	}
156f3515741SDavid Howells	return buf;
157f3515741SDavid Howells}
158f3515741SDavid HowellsEXPORT_SYMBOL(kmemdup_nul);
159f3515741SDavid Howells
160610a77e0SLi Zefan/**
161610a77e0SLi Zefan * memdup_user - duplicate memory region from user space
162610a77e0SLi Zefan *
163610a77e0SLi Zefan * @src: source address in user space
164610a77e0SLi Zefan * @len: number of bytes to copy
165610a77e0SLi Zefan *
166a862f68aSMike Rapoport * Return: an ERR_PTR() on failure.  Result is physically
16750fd2f29SAl Viro * contiguous, to be freed by kfree().
168610a77e0SLi Zefan */
169610a77e0SLi Zefanvoid *memdup_user(const void __user *src, size_t len)
170610a77e0SLi Zefan{
171610a77e0SLi Zefan	void *p;
172610a77e0SLi Zefan
1736c8fcc09SDaniel Vetter	p = kmalloc_track_caller(len, GFP_USER | __GFP_NOWARN);
174610a77e0SLi Zefan	if (!p)
175610a77e0SLi Zefan		return ERR_PTR(-ENOMEM);
176610a77e0SLi Zefan
177610a77e0SLi Zefan	if (copy_from_user(p, src, len)) {
178610a77e0SLi Zefan		kfree(p);
179610a77e0SLi Zefan		return ERR_PTR(-EFAULT);
180610a77e0SLi Zefan	}
181610a77e0SLi Zefan
182610a77e0SLi Zefan	return p;
183610a77e0SLi Zefan}
184610a77e0SLi ZefanEXPORT_SYMBOL(memdup_user);
185610a77e0SLi Zefan
18650fd2f29SAl Viro/**
18750fd2f29SAl Viro * vmemdup_user - duplicate memory region from user space
18850fd2f29SAl Viro *
18950fd2f29SAl Viro * @src: source address in user space
19050fd2f29SAl Viro * @len: number of bytes to copy
19150fd2f29SAl Viro *
192a862f68aSMike Rapoport * Return: an ERR_PTR() on failure.  Result may be not
19350fd2f29SAl Viro * physically contiguous.  Use kvfree() to free.
19450fd2f29SAl Viro */
19550fd2f29SAl Virovoid *vmemdup_user(const void __user *src, size_t len)
19650fd2f29SAl Viro{
19750fd2f29SAl Viro	void *p;
19850fd2f29SAl Viro
19950fd2f29SAl Viro	p = kvmalloc(len, GFP_USER);
20050fd2f29SAl Viro	if (!p)
20150fd2f29SAl Viro		return ERR_PTR(-ENOMEM);
20250fd2f29SAl Viro
20350fd2f29SAl Viro	if (copy_from_user(p, src, len)) {
20450fd2f29SAl Viro		kvfree(p);
20550fd2f29SAl Viro		return ERR_PTR(-EFAULT);
20650fd2f29SAl Viro	}
20750fd2f29SAl Viro
20850fd2f29SAl Viro	return p;
20950fd2f29SAl Viro}
21050fd2f29SAl ViroEXPORT_SYMBOL(vmemdup_user);
21150fd2f29SAl Viro
212b86181f1SMike Rapoport/**
21396840aa0SDavi Arnaut * strndup_user - duplicate an existing string from user space
21496840aa0SDavi Arnaut * @s: The string to duplicate
21596840aa0SDavi Arnaut * @n: Maximum number of bytes to copy, including the trailing NUL.
216a862f68aSMike Rapoport *
217e9145521SAndrew Morton * Return: newly allocated copy of @s or an ERR_PTR() in case of error
21896840aa0SDavi Arnaut */
21996840aa0SDavi Arnautchar *strndup_user(const char __user *s, long n)
22096840aa0SDavi Arnaut{
22196840aa0SDavi Arnaut	char *p;
22296840aa0SDavi Arnaut	long length;
22396840aa0SDavi Arnaut
22496840aa0SDavi Arnaut	length = strnlen_user(s, n);
22596840aa0SDavi Arnaut
22696840aa0SDavi Arnaut	if (!length)
22796840aa0SDavi Arnaut		return ERR_PTR(-EFAULT);
22896840aa0SDavi Arnaut
22996840aa0SDavi Arnaut	if (length > n)
23096840aa0SDavi Arnaut		return ERR_PTR(-EINVAL);
23196840aa0SDavi Arnaut
23290d74045SJulia Lawall	p = memdup_user(s, length);
23396840aa0SDavi Arnaut
23490d74045SJulia Lawall	if (IS_ERR(p))
23590d74045SJulia Lawall		return p;
23696840aa0SDavi Arnaut
23796840aa0SDavi Arnaut	p[length - 1] = '\0';
23896840aa0SDavi Arnaut
23996840aa0SDavi Arnaut	return p;
24096840aa0SDavi Arnaut}
24196840aa0SDavi ArnautEXPORT_SYMBOL(strndup_user);
24216d69265SAndrew Morton
243e9d408e1SAl Viro/**
244e9d408e1SAl Viro * memdup_user_nul - duplicate memory region from user space and NUL-terminate
245e9d408e1SAl Viro *
246e9d408e1SAl Viro * @src: source address in user space
247e9d408e1SAl Viro * @len: number of bytes to copy
248e9d408e1SAl Viro *
249a862f68aSMike Rapoport * Return: an ERR_PTR() on failure.
250e9d408e1SAl Viro */
251e9d408e1SAl Virovoid *memdup_user_nul(const void __user *src, size_t len)
252e9d408e1SAl Viro{
253e9d408e1SAl Viro	char *p;
254e9d408e1SAl Viro
255e9d408e1SAl Viro	/*
256e9d408e1SAl Viro	 * Always use GFP_KERNEL, since copy_from_user() can sleep and
257e9d408e1SAl Viro	 * cause pagefault, which makes it pointless to use GFP_NOFS
258e9d408e1SAl Viro	 * or GFP_ATOMIC.
259e9d408e1SAl Viro	 */
260e9d408e1SAl Viro	p = kmalloc_track_caller(len + 1, GFP_KERNEL);
261e9d408e1SAl Viro	if (!p)
262e9d408e1SAl Viro		return ERR_PTR(-ENOMEM);
263e9d408e1SAl Viro
264e9d408e1SAl Viro	if (copy_from_user(p, src, len)) {
265e9d408e1SAl Viro		kfree(p);
266e9d408e1SAl Viro		return ERR_PTR(-EFAULT);
267e9d408e1SAl Viro	}
268e9d408e1SAl Viro	p[len] = '\0';
269e9d408e1SAl Viro
270e9d408e1SAl Viro	return p;
271e9d408e1SAl Viro}
272e9d408e1SAl ViroEXPORT_SYMBOL(memdup_user_nul);
273e9d408e1SAl Viro
2746038def0SNamhyung Kimvoid __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
275aba6dfb7SWei Yang		struct vm_area_struct *prev)
2766038def0SNamhyung Kim{
2776038def0SNamhyung Kim	struct vm_area_struct *next;
2786038def0SNamhyung Kim
2796038def0SNamhyung Kim	vma->vm_prev = prev;
2806038def0SNamhyung Kim	if (prev) {
2816038def0SNamhyung Kim		next = prev->vm_next;
2826038def0SNamhyung Kim		prev->vm_next = vma;
2836038def0SNamhyung Kim	} else {
284aba6dfb7SWei Yang		next = mm->mmap;
2856038def0SNamhyung Kim		mm->mmap = vma;
2866038def0SNamhyung Kim	}
2876038def0SNamhyung Kim	vma->vm_next = next;
2886038def0SNamhyung Kim	if (next)
2896038def0SNamhyung Kim		next->vm_prev = vma;
2906038def0SNamhyung Kim}
2916038def0SNamhyung Kim
2921b9fc5b2SWei Yangvoid __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma)
2931b9fc5b2SWei Yang{
2941b9fc5b2SWei Yang	struct vm_area_struct *prev, *next;
2951b9fc5b2SWei Yang
2961b9fc5b2SWei Yang	next = vma->vm_next;
2971b9fc5b2SWei Yang	prev = vma->vm_prev;
2981b9fc5b2SWei Yang	if (prev)
2991b9fc5b2SWei Yang		prev->vm_next = next;
3001b9fc5b2SWei Yang	else
3011b9fc5b2SWei Yang		mm->mmap = next;
3021b9fc5b2SWei Yang	if (next)
3031b9fc5b2SWei Yang		next->vm_prev = prev;
3041b9fc5b2SWei Yang}
3051b9fc5b2SWei Yang
306b7643757SSiddhesh Poyarekar/* Check if the vma is being used as a stack by this task */
307d17af505SAndy Lutomirskiint vma_is_stack_for_current(struct vm_area_struct *vma)
308b7643757SSiddhesh Poyarekar{
309d17af505SAndy Lutomirski	struct task_struct * __maybe_unused t = current;
310d17af505SAndy Lutomirski
311b7643757SSiddhesh Poyarekar	return (vma->vm_start <= KSTK_ESP(t) && vma->vm_end >= KSTK_ESP(t));
312b7643757SSiddhesh Poyarekar}
313b7643757SSiddhesh Poyarekar
314295992fbSChristian König/*
315295992fbSChristian König * Change backing file, only valid to use during initial VMA setup.
316295992fbSChristian König */
317295992fbSChristian Königvoid vma_set_file(struct vm_area_struct *vma, struct file *file)
318295992fbSChristian König{
319295992fbSChristian König	/* Changing an anonymous vma with this is illegal */
320295992fbSChristian König	get_file(file);
321295992fbSChristian König	swap(vma->vm_file, file);
322