namei.c revision 794ebcea
1// SPDX-License-Identifier: GPL-2.0
2/*
3 *  linux/fs/namei.c
4 *
5 *  Copyright (C) 1991, 1992  Linus Torvalds
6 */
7
8/*
9 * Some corrections by tytso.
10 */
11
12/* [Feb 1997 T. Schoebel-Theuer] Complete rewrite of the pathname
13 * lookup logic.
14 */
15/* [Feb-Apr 2000, AV] Rewrite to the new namespace architecture.
16 */
17
18#include <linux/init.h>
19#include <linux/export.h>
20#include <linux/kernel.h>
21#include <linux/slab.h>
22#include <linux/fs.h>
23#include <linux/namei.h>
24#include <linux/pagemap.h>
25#include <linux/fsnotify.h>
26#include <linux/personality.h>
27#include <linux/security.h>
28#include <linux/ima.h>
29#include <linux/syscalls.h>
30#include <linux/mount.h>
31#include <linux/audit.h>
32#include <linux/capability.h>
33#include <linux/file.h>
34#include <linux/fcntl.h>
35#include <linux/device_cgroup.h>
36#include <linux/fs_struct.h>
37#include <linux/posix_acl.h>
38#include <linux/hash.h>
39#include <linux/bitops.h>
40#include <linux/init_task.h>
41#include <linux/uaccess.h>
42
43#include "internal.h"
44#include "mount.h"
45
46/* [Feb-1997 T. Schoebel-Theuer]
47 * Fundamental changes in the pathname lookup mechanisms (namei)
48 * were necessary because of omirr.  The reason is that omirr needs
49 * to know the _real_ pathname, not the user-supplied one, in case
50 * of symlinks (and also when transname replacements occur).
51 *
52 * The new code replaces the old recursive symlink resolution with
53 * an iterative one (in case of non-nested symlink chains).  It does
54 * this with calls to <fs>_follow_link().
55 * As a side effect, dir_namei(), _namei() and follow_link() are now
56 * replaced with a single function lookup_dentry() that can handle all
57 * the special cases of the former code.
58 *
59 * With the new dcache, the pathname is stored at each inode, at least as
60 * long as the refcount of the inode is positive.  As a side effect, the
61 * size of the dcache depends on the inode cache and thus is dynamic.
62 *
63 * [29-Apr-1998 C. Scott Ananian] Updated above description of symlink
64 * resolution to correspond with current state of the code.
65 *
66 * Note that the symlink resolution is not *completely* iterative.
67 * There is still a significant amount of tail- and mid- recursion in
68 * the algorithm.  Also, note that <fs>_readlink() is not used in
69 * lookup_dentry(): lookup_dentry() on the result of <fs>_readlink()
70 * may return different results than <fs>_follow_link().  Many virtual
71 * filesystems (including /proc) exhibit this behavior.
72 */
73
74/* [24-Feb-97 T. Schoebel-Theuer] Side effects caused by new implementation:
75 * New symlink semantics: when open() is called with flags O_CREAT | O_EXCL
76 * and the name already exists in form of a symlink, try to create the new
77 * name indicated by the symlink. The old code always complained that the
78 * name already exists, due to not following the symlink even if its target
79 * is nonexistent.  The new semantics affects also mknod() and link() when
80 * the name is a symlink pointing to a non-existent name.
81 *
82 * I don't know which semantics is the right one, since I have no access
83 * to standards. But I found by trial that HP-UX 9.0 has the full "new"
84 * semantics implemented, while SunOS 4.1.1 and Solaris (SunOS 5.4) have the
85 * "old" one. Personally, I think the new semantics is much more logical.
86 * Note that "ln old new" where "new" is a symlink pointing to a non-existing
87 * file does succeed in both HP-UX and SunOs, but not in Solaris
88 * and in the old Linux semantics.
89 */
90
91/* [16-Dec-97 Kevin Buhr] For security reasons, we change some symlink
92 * semantics.  See the comments in "open_namei" and "do_link" below.
93 *
94 * [10-Sep-98 Alan Modra] Another symlink change.
95 */
96
97/* [Feb-Apr 2000 AV] Complete rewrite. Rules for symlinks:
98 *	inside the path - always follow.
99 *	in the last component in creation/removal/renaming - never follow.
100 *	if LOOKUP_FOLLOW passed - follow.
101 *	if the pathname has trailing slashes - follow.
102 *	otherwise - don't follow.
103 * (applied in that order).
104 *
105 * [Jun 2000 AV] Inconsistent behaviour of open() in case if flags==O_CREAT
106 * restored for 2.4. This is the last surviving part of old 4.2BSD bug.
107 * During the 2.4 we need to fix the userland stuff depending on it -
108 * hopefully we will be able to get rid of that wart in 2.5. So far only
109 * XEmacs seems to be relying on it...
110 */
111/*
112 * [Sep 2001 AV] Single-semaphore locking scheme (kudos to David Holland)
113 * implemented.  Let's see if raised priority of ->s_vfs_rename_mutex gives
114 * any extra contention...
115 */
116
117/* In order to reduce some races, while at the same time doing additional
118 * checking and hopefully speeding things up, we copy filenames to the
119 * kernel data space before using them..
120 *
121 * POSIX.1 2.4: an empty pathname is invalid (ENOENT).
122 * PATH_MAX includes the nul terminator --RR.
123 */
124
125#define EMBEDDED_NAME_MAX	(PATH_MAX - offsetof(struct filename, iname))
126
127struct filename *
128getname_flags(const char __user *filename, int flags, int *empty)
129{
130	struct filename *result;
131	char *kname;
132	int len;
133
134	result = audit_reusename(filename);
135	if (result)
136		return result;
137
138	result = __getname();
139	if (unlikely(!result))
140		return ERR_PTR(-ENOMEM);
141
142	/*
143	 * First, try to embed the struct filename inside the names_cache
144	 * allocation
145	 */
146	kname = (char *)result->iname;
147	result->name = kname;
148
149	len = strncpy_from_user(kname, filename, EMBEDDED_NAME_MAX);
150	if (unlikely(len < 0)) {
151		__putname(result);
152		return ERR_PTR(len);
153	}
154
155	/*
156	 * Uh-oh. We have a name that's approaching PATH_MAX. Allocate a
157	 * separate struct filename so we can dedicate the entire
158	 * names_cache allocation for the pathname, and re-do the copy from
159	 * userland.
160	 */
161	if (unlikely(len == EMBEDDED_NAME_MAX)) {
162		const size_t size = offsetof(struct filename, iname[1]);
163		kname = (char *)result;
164
165		/*
166		 * size is chosen that way we to guarantee that
167		 * result->iname[0] is within the same object and that
168		 * kname can't be equal to result->iname, no matter what.
169		 */
170		result = kzalloc(size, GFP_KERNEL);
171		if (unlikely(!result)) {
172			__putname(kname);
173			return ERR_PTR(-ENOMEM);
174		}
175		result->name = kname;
176		len = strncpy_from_user(kname, filename, PATH_MAX);
177		if (unlikely(len < 0)) {
178			__putname(kname);
179			kfree(result);
180			return ERR_PTR(len);
181		}
182		if (unlikely(len == PATH_MAX)) {
183			__putname(kname);
184			kfree(result);
185			return ERR_PTR(-ENAMETOOLONG);
186		}
187	}
188
189	result->refcnt = 1;
190	/* The empty path is special. */
191	if (unlikely(!len)) {
192		if (empty)
193			*empty = 1;
194		if (!(flags & LOOKUP_EMPTY)) {
195			putname(result);
196			return ERR_PTR(-ENOENT);
197		}
198	}
199
200	result->uptr = filename;
201	result->aname = NULL;
202	audit_getname(result);
203	return result;
204}
205
206struct filename *
207getname_uflags(const char __user *filename, int uflags)
208{
209	int flags = (uflags & AT_EMPTY_PATH) ? LOOKUP_EMPTY : 0;
210
211	return getname_flags(filename, flags, NULL);
212}
213
214struct filename *
215getname(const char __user * filename)
216{
217	return getname_flags(filename, 0, NULL);
218}
219
220struct filename *
221getname_kernel(const char * filename)
222{
223	struct filename *result;
224	int len = strlen(filename) + 1;
225
226	result = __getname();
227	if (unlikely(!result))
228		return ERR_PTR(-ENOMEM);
229
230	if (len <= EMBEDDED_NAME_MAX) {
231		result->name = (char *)result->iname;
232	} else if (len <= PATH_MAX) {
233		const size_t size = offsetof(struct filename, iname[1]);
234		struct filename *tmp;
235
236		tmp = kmalloc(size, GFP_KERNEL);
237		if (unlikely(!tmp)) {
238			__putname(result);
239			return ERR_PTR(-ENOMEM);
240		}
241		tmp->name = (char *)result;
242		result = tmp;
243	} else {
244		__putname(result);
245		return ERR_PTR(-ENAMETOOLONG);
246	}
247	memcpy((char *)result->name, filename, len);
248	result->uptr = NULL;
249	result->aname = NULL;
250	result->refcnt = 1;
251	audit_getname(result);
252
253	return result;
254}
255
256void putname(struct filename *name)
257{
258	if (IS_ERR_OR_NULL(name))
259		return;
260
261	BUG_ON(name->refcnt <= 0);
262
263	if (--name->refcnt > 0)
264		return;
265
266	if (name->name != name->iname) {
267		__putname(name->name);
268		kfree(name);
269	} else
270		__putname(name);
271}
272
273/**
274 * check_acl - perform ACL permission checking
275 * @mnt_userns:	user namespace of the mount the inode was found from
276 * @inode:	inode to check permissions on
277 * @mask:	right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...)
278 *
279 * This function performs the ACL permission checking. Since this function
280 * retrieve POSIX acls it needs to know whether it is called from a blocking or
281 * non-blocking context and thus cares about the MAY_NOT_BLOCK bit.
282 *
283 * If the inode has been found through an idmapped mount the user namespace of
284 * the vfsmount must be passed through @mnt_userns. This function will then take
285 * care to map the inode according to @mnt_userns before checking permissions.
286 * On non-idmapped mounts or if permission checking is to be performed on the
287 * raw inode simply passs init_user_ns.
288 */
289static int check_acl(struct user_namespace *mnt_userns,
290		     struct inode *inode, int mask)
291{
292#ifdef CONFIG_FS_POSIX_ACL
293	struct posix_acl *acl;
294
295	if (mask & MAY_NOT_BLOCK) {
296		acl = get_cached_acl_rcu(inode, ACL_TYPE_ACCESS);
297	        if (!acl)
298	                return -EAGAIN;
299		/* no ->get_acl() calls in RCU mode... */
300		if (is_uncached_acl(acl))
301			return -ECHILD;
302	        return posix_acl_permission(mnt_userns, inode, acl, mask);
303	}
304
305	acl = get_acl(inode, ACL_TYPE_ACCESS);
306	if (IS_ERR(acl))
307		return PTR_ERR(acl);
308	if (acl) {
309	        int error = posix_acl_permission(mnt_userns, inode, acl, mask);
310	        posix_acl_release(acl);
311	        return error;
312	}
313#endif
314
315	return -EAGAIN;
316}
317
318/**
319 * acl_permission_check - perform basic UNIX permission checking
320 * @mnt_userns:	user namespace of the mount the inode was found from
321 * @inode:	inode to check permissions on
322 * @mask:	right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...)
323 *
324 * This function performs the basic UNIX permission checking. Since this
325 * function may retrieve POSIX acls it needs to know whether it is called from a
326 * blocking or non-blocking context and thus cares about the MAY_NOT_BLOCK bit.
327 *
328 * If the inode has been found through an idmapped mount the user namespace of
329 * the vfsmount must be passed through @mnt_userns. This function will then take
330 * care to map the inode according to @mnt_userns before checking permissions.
331 * On non-idmapped mounts or if permission checking is to be performed on the
332 * raw inode simply passs init_user_ns.
333 */
334static int acl_permission_check(struct user_namespace *mnt_userns,
335				struct inode *inode, int mask)
336{
337	unsigned int mode = inode->i_mode;
338	kuid_t i_uid;
339
340	/* Are we the owner? If so, ACL's don't matter */
341	i_uid = i_uid_into_mnt(mnt_userns, inode);
342	if (likely(uid_eq(current_fsuid(), i_uid))) {
343		mask &= 7;
344		mode >>= 6;
345		return (mask & ~mode) ? -EACCES : 0;
346	}
347
348	/* Do we have ACL's? */
349	if (IS_POSIXACL(inode) && (mode & S_IRWXG)) {
350		int error = check_acl(mnt_userns, inode, mask);
351		if (error != -EAGAIN)
352			return error;
353	}
354
355	/* Only RWX matters for group/other mode bits */
356	mask &= 7;
357
358	/*
359	 * Are the group permissions different from
360	 * the other permissions in the bits we care
361	 * about? Need to check group ownership if so.
362	 */
363	if (mask & (mode ^ (mode >> 3))) {
364		kgid_t kgid = i_gid_into_mnt(mnt_userns, inode);
365		if (in_group_p(kgid))
366			mode >>= 3;
367	}
368
369	/* Bits in 'mode' clear that we require? */
370	return (mask & ~mode) ? -EACCES : 0;
371}
372
373/**
374 * generic_permission -  check for access rights on a Posix-like filesystem
375 * @mnt_userns:	user namespace of the mount the inode was found from
376 * @inode:	inode to check access rights for
377 * @mask:	right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC,
378 *		%MAY_NOT_BLOCK ...)
379 *
380 * Used to check for read/write/execute permissions on a file.
381 * We use "fsuid" for this, letting us set arbitrary permissions
382 * for filesystem access without changing the "normal" uids which
383 * are used for other things.
384 *
385 * generic_permission is rcu-walk aware. It returns -ECHILD in case an rcu-walk
386 * request cannot be satisfied (eg. requires blocking or too much complexity).
387 * It would then be called again in ref-walk mode.
388 *
389 * If the inode has been found through an idmapped mount the user namespace of
390 * the vfsmount must be passed through @mnt_userns. This function will then take
391 * care to map the inode according to @mnt_userns before checking permissions.
392 * On non-idmapped mounts or if permission checking is to be performed on the
393 * raw inode simply passs init_user_ns.
394 */
395int generic_permission(struct user_namespace *mnt_userns, struct inode *inode,
396		       int mask)
397{
398	int ret;
399
400	/*
401	 * Do the basic permission checks.
402	 */
403	ret = acl_permission_check(mnt_userns, inode, mask);
404	if (ret != -EACCES)
405		return ret;
406
407	if (S_ISDIR(inode->i_mode)) {
408		/* DACs are overridable for directories */
409		if (!(mask & MAY_WRITE))
410			if (capable_wrt_inode_uidgid(mnt_userns, inode,
411						     CAP_DAC_READ_SEARCH))
412				return 0;
413		if (capable_wrt_inode_uidgid(mnt_userns, inode,
414					     CAP_DAC_OVERRIDE))
415			return 0;
416		return -EACCES;
417	}
418
419	/*
420	 * Searching includes executable on directories, else just read.
421	 */
422	mask &= MAY_READ | MAY_WRITE | MAY_EXEC;
423	if (mask == MAY_READ)
424		if (capable_wrt_inode_uidgid(mnt_userns, inode,
425					     CAP_DAC_READ_SEARCH))
426			return 0;
427	/*
428	 * Read/write DACs are always overridable.
429	 * Executable DACs are overridable when there is
430	 * at least one exec bit set.
431	 */
432	if (!(mask & MAY_EXEC) || (inode->i_mode & S_IXUGO))
433		if (capable_wrt_inode_uidgid(mnt_userns, inode,
434					     CAP_DAC_OVERRIDE))
435			return 0;
436
437	return -EACCES;
438}
439EXPORT_SYMBOL(generic_permission);
440
441/**
442 * do_inode_permission - UNIX permission checking
443 * @mnt_userns:	user namespace of the mount the inode was found from
444 * @inode:	inode to check permissions on
445 * @mask:	right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...)
446 *
447 * We _really_ want to just do "generic_permission()" without
448 * even looking at the inode->i_op values. So we keep a cache
449 * flag in inode->i_opflags, that says "this has not special
450 * permission function, use the fast case".
451 */
452static inline int do_inode_permission(struct user_namespace *mnt_userns,
453				      struct inode *inode, int mask)
454{
455	if (unlikely(!(inode->i_opflags & IOP_FASTPERM))) {
456		if (likely(inode->i_op->permission))
457			return inode->i_op->permission(mnt_userns, inode, mask);
458
459		/* This gets set once for the inode lifetime */
460		spin_lock(&inode->i_lock);
461		inode->i_opflags |= IOP_FASTPERM;
462		spin_unlock(&inode->i_lock);
463	}
464	return generic_permission(mnt_userns, inode, mask);
465}
466
467/**
468 * sb_permission - Check superblock-level permissions
469 * @sb: Superblock of inode to check permission on
470 * @inode: Inode to check permission on
471 * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
472 *
473 * Separate out file-system wide checks from inode-specific permission checks.
474 */
475static int sb_permission(struct super_block *sb, struct inode *inode, int mask)
476{
477	if (unlikely(mask & MAY_WRITE)) {
478		umode_t mode = inode->i_mode;
479
480		/* Nobody gets write access to a read-only fs. */
481		if (sb_rdonly(sb) && (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode)))
482			return -EROFS;
483	}
484	return 0;
485}
486
487/**
488 * inode_permission - Check for access rights to a given inode
489 * @mnt_userns:	User namespace of the mount the inode was found from
490 * @inode:	Inode to check permission on
491 * @mask:	Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
492 *
493 * Check for read/write/execute permissions on an inode.  We use fs[ug]id for
494 * this, letting us set arbitrary permissions for filesystem access without
495 * changing the "normal" UIDs which are used for other things.
496 *
497 * When checking for MAY_APPEND, MAY_WRITE must also be set in @mask.
498 */
499int inode_permission(struct user_namespace *mnt_userns,
500		     struct inode *inode, int mask)
501{
502	int retval;
503
504	retval = sb_permission(inode->i_sb, inode, mask);
505	if (retval)
506		return retval;
507
508	if (unlikely(mask & MAY_WRITE)) {
509		/*
510		 * Nobody gets write access to an immutable file.
511		 */
512		if (IS_IMMUTABLE(inode))
513			return -EPERM;
514
515		/*
516		 * Updating mtime will likely cause i_uid and i_gid to be
517		 * written back improperly if their true value is unknown
518		 * to the vfs.
519		 */
520		if (HAS_UNMAPPED_ID(mnt_userns, inode))
521			return -EACCES;
522	}
523
524	retval = do_inode_permission(mnt_userns, inode, mask);
525	if (retval)
526		return retval;
527
528	retval = devcgroup_inode_permission(inode, mask);
529	if (retval)
530		return retval;
531
532	return security_inode_permission(inode, mask);
533}
534EXPORT_SYMBOL(inode_permission);
535
536/**
537 * path_get - get a reference to a path
538 * @path: path to get the reference to
539 *
540 * Given a path increment the reference count to the dentry and the vfsmount.
541 */
542void path_get(const struct path *path)
543{
544	mntget(path->mnt);
545	dget(path->dentry);
546}
547EXPORT_SYMBOL(path_get);
548
549/**
550 * path_put - put a reference to a path
551 * @path: path to put the reference to
552 *
553 * Given a path decrement the reference count to the dentry and the vfsmount.
554 */
555void path_put(const struct path *path)
556{
557	dput(path->dentry);
558	mntput(path->mnt);
559}
560EXPORT_SYMBOL(path_put);
561
562#define EMBEDDED_LEVELS 2
563struct nameidata {
564	struct path	path;
565	struct qstr	last;
566	struct path	root;
567	struct inode	*inode; /* path.dentry.d_inode */
568	unsigned int	flags, state;
569	unsigned	seq, m_seq, r_seq;
570	int		last_type;
571	unsigned	depth;
572	int		total_link_count;
573	struct saved {
574		struct path link;
575		struct delayed_call done;
576		const char *name;
577		unsigned seq;
578	} *stack, internal[EMBEDDED_LEVELS];
579	struct filename	*name;
580	struct nameidata *saved;
581	unsigned	root_seq;
582	int		dfd;
583	kuid_t		dir_uid;
584	umode_t		dir_mode;
585} __randomize_layout;
586
587#define ND_ROOT_PRESET 1
588#define ND_ROOT_GRABBED 2
589#define ND_JUMPED 4
590
591static void __set_nameidata(struct nameidata *p, int dfd, struct filename *name)
592{
593	struct nameidata *old = current->nameidata;
594	p->stack = p->internal;
595	p->depth = 0;
596	p->dfd = dfd;
597	p->name = name;
598	p->path.mnt = NULL;
599	p->path.dentry = NULL;
600	p->total_link_count = old ? old->total_link_count : 0;
601	p->saved = old;
602	current->nameidata = p;
603}
604
605static inline void set_nameidata(struct nameidata *p, int dfd, struct filename *name,
606			  const struct path *root)
607{
608	__set_nameidata(p, dfd, name);
609	p->state = 0;
610	if (unlikely(root)) {
611		p->state = ND_ROOT_PRESET;
612		p->root = *root;
613	}
614}
615
616static void restore_nameidata(void)
617{
618	struct nameidata *now = current->nameidata, *old = now->saved;
619
620	current->nameidata = old;
621	if (old)
622		old->total_link_count = now->total_link_count;
623	if (now->stack != now->internal)
624		kfree(now->stack);
625}
626
627static bool nd_alloc_stack(struct nameidata *nd)
628{
629	struct saved *p;
630
631	p= kmalloc_array(MAXSYMLINKS, sizeof(struct saved),
632			 nd->flags & LOOKUP_RCU ? GFP_ATOMIC : GFP_KERNEL);
633	if (unlikely(!p))
634		return false;
635	memcpy(p, nd->internal, sizeof(nd->internal));
636	nd->stack = p;
637	return true;
638}
639
640/**
641 * path_connected - Verify that a dentry is below mnt.mnt_root
642 *
643 * Rename can sometimes move a file or directory outside of a bind
644 * mount, path_connected allows those cases to be detected.
645 */
646static bool path_connected(struct vfsmount *mnt, struct dentry *dentry)
647{
648	struct super_block *sb = mnt->mnt_sb;
649
650	/* Bind mounts can have disconnected paths */
651	if (mnt->mnt_root == sb->s_root)
652		return true;
653
654	return is_subdir(dentry, mnt->mnt_root);
655}
656
657static void drop_links(struct nameidata *nd)
658{
659	int i = nd->depth;
660	while (i--) {
661		struct saved *last = nd->stack + i;
662		do_delayed_call(&last->done);
663		clear_delayed_call(&last->done);
664	}
665}
666
667static void terminate_walk(struct nameidata *nd)
668{
669	drop_links(nd);
670	if (!(nd->flags & LOOKUP_RCU)) {
671		int i;
672		path_put(&nd->path);
673		for (i = 0; i < nd->depth; i++)
674			path_put(&nd->stack[i].link);
675		if (nd->state & ND_ROOT_GRABBED) {
676			path_put(&nd->root);
677			nd->state &= ~ND_ROOT_GRABBED;
678		}
679	} else {
680		nd->flags &= ~LOOKUP_RCU;
681		rcu_read_unlock();
682	}
683	nd->depth = 0;
684	nd->path.mnt = NULL;
685	nd->path.dentry = NULL;
686}
687
688/* path_put is needed afterwards regardless of success or failure */
689static bool __legitimize_path(struct path *path, unsigned seq, unsigned mseq)
690{
691	int res = __legitimize_mnt(path->mnt, mseq);
692	if (unlikely(res)) {
693		if (res > 0)
694			path->mnt = NULL;
695		path->dentry = NULL;
696		return false;
697	}
698	if (unlikely(!lockref_get_not_dead(&path->dentry->d_lockref))) {
699		path->dentry = NULL;
700		return false;
701	}
702	return !read_seqcount_retry(&path->dentry->d_seq, seq);
703}
704
705static inline bool legitimize_path(struct nameidata *nd,
706			    struct path *path, unsigned seq)
707{
708	return __legitimize_path(path, seq, nd->m_seq);
709}
710
711static bool legitimize_links(struct nameidata *nd)
712{
713	int i;
714	if (unlikely(nd->flags & LOOKUP_CACHED)) {
715		drop_links(nd);
716		nd->depth = 0;
717		return false;
718	}
719	for (i = 0; i < nd->depth; i++) {
720		struct saved *last = nd->stack + i;
721		if (unlikely(!legitimize_path(nd, &last->link, last->seq))) {
722			drop_links(nd);
723			nd->depth = i + 1;
724			return false;
725		}
726	}
727	return true;
728}
729
730static bool legitimize_root(struct nameidata *nd)
731{
732	/*
733	 * For scoped-lookups (where nd->root has been zeroed), we need to
734	 * restart the whole lookup from scratch -- because set_root() is wrong
735	 * for these lookups (nd->dfd is the root, not the filesystem root).
736	 */
737	if (!nd->root.mnt && (nd->flags & LOOKUP_IS_SCOPED))
738		return false;
739	/* Nothing to do if nd->root is zero or is managed by the VFS user. */
740	if (!nd->root.mnt || (nd->state & ND_ROOT_PRESET))
741		return true;
742	nd->state |= ND_ROOT_GRABBED;
743	return legitimize_path(nd, &nd->root, nd->root_seq);
744}
745
746/*
747 * Path walking has 2 modes, rcu-walk and ref-walk (see
748 * Documentation/filesystems/path-lookup.txt).  In situations when we can't
749 * continue in RCU mode, we attempt to drop out of rcu-walk mode and grab
750 * normal reference counts on dentries and vfsmounts to transition to ref-walk
751 * mode.  Refcounts are grabbed at the last known good point before rcu-walk
752 * got stuck, so ref-walk may continue from there. If this is not successful
753 * (eg. a seqcount has changed), then failure is returned and it's up to caller
754 * to restart the path walk from the beginning in ref-walk mode.
755 */
756
757/**
758 * try_to_unlazy - try to switch to ref-walk mode.
759 * @nd: nameidata pathwalk data
760 * Returns: true on success, false on failure
761 *
762 * try_to_unlazy attempts to legitimize the current nd->path and nd->root
763 * for ref-walk mode.
764 * Must be called from rcu-walk context.
765 * Nothing should touch nameidata between try_to_unlazy() failure and
766 * terminate_walk().
767 */
768static bool try_to_unlazy(struct nameidata *nd)
769{
770	struct dentry *parent = nd->path.dentry;
771
772	BUG_ON(!(nd->flags & LOOKUP_RCU));
773
774	nd->flags &= ~LOOKUP_RCU;
775	if (unlikely(!legitimize_links(nd)))
776		goto out1;
777	if (unlikely(!legitimize_path(nd, &nd->path, nd->seq)))
778		goto out;
779	if (unlikely(!legitimize_root(nd)))
780		goto out;
781	rcu_read_unlock();
782	BUG_ON(nd->inode != parent->d_inode);
783	return true;
784
785out1:
786	nd->path.mnt = NULL;
787	nd->path.dentry = NULL;
788out:
789	rcu_read_unlock();
790	return false;
791}
792
793/**
794 * try_to_unlazy_next - try to switch to ref-walk mode.
795 * @nd: nameidata pathwalk data
796 * @dentry: next dentry to step into
797 * @seq: seq number to check @dentry against
798 * Returns: true on success, false on failure
799 *
800 * Similar to to try_to_unlazy(), but here we have the next dentry already
801 * picked by rcu-walk and want to legitimize that in addition to the current
802 * nd->path and nd->root for ref-walk mode.  Must be called from rcu-walk context.
803 * Nothing should touch nameidata between try_to_unlazy_next() failure and
804 * terminate_walk().
805 */
806static bool try_to_unlazy_next(struct nameidata *nd, struct dentry *dentry, unsigned seq)
807{
808	BUG_ON(!(nd->flags & LOOKUP_RCU));
809
810	nd->flags &= ~LOOKUP_RCU;
811	if (unlikely(!legitimize_links(nd)))
812		goto out2;
813	if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq)))
814		goto out2;
815	if (unlikely(!lockref_get_not_dead(&nd->path.dentry->d_lockref)))
816		goto out1;
817
818	/*
819	 * We need to move both the parent and the dentry from the RCU domain
820	 * to be properly refcounted. And the sequence number in the dentry
821	 * validates *both* dentry counters, since we checked the sequence
822	 * number of the parent after we got the child sequence number. So we
823	 * know the parent must still be valid if the child sequence number is
824	 */
825	if (unlikely(!lockref_get_not_dead(&dentry->d_lockref)))
826		goto out;
827	if (unlikely(read_seqcount_retry(&dentry->d_seq, seq)))
828		goto out_dput;
829	/*
830	 * Sequence counts matched. Now make sure that the root is
831	 * still valid and get it if required.
832	 */
833	if (unlikely(!legitimize_root(nd)))
834		goto out_dput;
835	rcu_read_unlock();
836	return true;
837
838out2:
839	nd->path.mnt = NULL;
840out1:
841	nd->path.dentry = NULL;
842out:
843	rcu_read_unlock();
844	return false;
845out_dput:
846	rcu_read_unlock();
847	dput(dentry);
848	return false;
849}
850
851static inline int d_revalidate(struct dentry *dentry, unsigned int flags)
852{
853	if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE))
854		return dentry->d_op->d_revalidate(dentry, flags);
855	else
856		return 1;
857}
858
859/**
860 * complete_walk - successful completion of path walk
861 * @nd:  pointer nameidata
862 *
863 * If we had been in RCU mode, drop out of it and legitimize nd->path.
864 * Revalidate the final result, unless we'd already done that during
865 * the path walk or the filesystem doesn't ask for it.  Return 0 on
866 * success, -error on failure.  In case of failure caller does not
867 * need to drop nd->path.
868 */
869static int complete_walk(struct nameidata *nd)
870{
871	struct dentry *dentry = nd->path.dentry;
872	int status;
873
874	if (nd->flags & LOOKUP_RCU) {
875		/*
876		 * We don't want to zero nd->root for scoped-lookups or
877		 * externally-managed nd->root.
878		 */
879		if (!(nd->state & ND_ROOT_PRESET))
880			if (!(nd->flags & LOOKUP_IS_SCOPED))
881				nd->root.mnt = NULL;
882		nd->flags &= ~LOOKUP_CACHED;
883		if (!try_to_unlazy(nd))
884			return -ECHILD;
885	}
886
887	if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) {
888		/*
889		 * While the guarantee of LOOKUP_IS_SCOPED is (roughly) "don't
890		 * ever step outside the root during lookup" and should already
891		 * be guaranteed by the rest of namei, we want to avoid a namei
892		 * BUG resulting in userspace being given a path that was not
893		 * scoped within the root at some point during the lookup.
894		 *
895		 * So, do a final sanity-check to make sure that in the
896		 * worst-case scenario (a complete bypass of LOOKUP_IS_SCOPED)
897		 * we won't silently return an fd completely outside of the
898		 * requested root to userspace.
899		 *
900		 * Userspace could move the path outside the root after this
901		 * check, but as discussed elsewhere this is not a concern (the
902		 * resolved file was inside the root at some point).
903		 */
904		if (!path_is_under(&nd->path, &nd->root))
905			return -EXDEV;
906	}
907
908	if (likely(!(nd->state & ND_JUMPED)))
909		return 0;
910
911	if (likely(!(dentry->d_flags & DCACHE_OP_WEAK_REVALIDATE)))
912		return 0;
913
914	status = dentry->d_op->d_weak_revalidate(dentry, nd->flags);
915	if (status > 0)
916		return 0;
917
918	if (!status)
919		status = -ESTALE;
920
921	return status;
922}
923
924static int set_root(struct nameidata *nd)
925{
926	struct fs_struct *fs = current->fs;
927
928	/*
929	 * Jumping to the real root in a scoped-lookup is a BUG in namei, but we
930	 * still have to ensure it doesn't happen because it will cause a breakout
931	 * from the dirfd.
932	 */
933	if (WARN_ON(nd->flags & LOOKUP_IS_SCOPED))
934		return -ENOTRECOVERABLE;
935
936	if (nd->flags & LOOKUP_RCU) {
937		unsigned seq;
938
939		do {
940			seq = read_seqcount_begin(&fs->seq);
941			nd->root = fs->root;
942			nd->root_seq = __read_seqcount_begin(&nd->root.dentry->d_seq);
943		} while (read_seqcount_retry(&fs->seq, seq));
944	} else {
945		get_fs_root(fs, &nd->root);
946		nd->state |= ND_ROOT_GRABBED;
947	}
948	return 0;
949}
950
951static int nd_jump_root(struct nameidata *nd)
952{
953	if (unlikely(nd->flags & LOOKUP_BENEATH))
954		return -EXDEV;
955	if (unlikely(nd->flags & LOOKUP_NO_XDEV)) {
956		/* Absolute path arguments to path_init() are allowed. */
957		if (nd->path.mnt != NULL && nd->path.mnt != nd->root.mnt)
958			return -EXDEV;
959	}
960	if (!nd->root.mnt) {
961		int error = set_root(nd);
962		if (error)
963			return error;
964	}
965	if (nd->flags & LOOKUP_RCU) {
966		struct dentry *d;
967		nd->path = nd->root;
968		d = nd->path.dentry;
969		nd->inode = d->d_inode;
970		nd->seq = nd->root_seq;
971		if (unlikely(read_seqcount_retry(&d->d_seq, nd->seq)))
972			return -ECHILD;
973	} else {
974		path_put(&nd->path);
975		nd->path = nd->root;
976		path_get(&nd->path);
977		nd->inode = nd->path.dentry->d_inode;
978	}
979	nd->state |= ND_JUMPED;
980	return 0;
981}
982
983/*
984 * Helper to directly jump to a known parsed path from ->get_link,
985 * caller must have taken a reference to path beforehand.
986 */
987int nd_jump_link(struct path *path)
988{
989	int error = -ELOOP;
990	struct nameidata *nd = current->nameidata;
991
992	if (unlikely(nd->flags & LOOKUP_NO_MAGICLINKS))
993		goto err;
994
995	error = -EXDEV;
996	if (unlikely(nd->flags & LOOKUP_NO_XDEV)) {
997		if (nd->path.mnt != path->mnt)
998			goto err;
999	}
1000	/* Not currently safe for scoped-lookups. */
1001	if (unlikely(nd->flags & LOOKUP_IS_SCOPED))
1002		goto err;
1003
1004	path_put(&nd->path);
1005	nd->path = *path;
1006	nd->inode = nd->path.dentry->d_inode;
1007	nd->state |= ND_JUMPED;
1008	return 0;
1009
1010err:
1011	path_put(path);
1012	return error;
1013}
1014
1015static inline void put_link(struct nameidata *nd)
1016{
1017	struct saved *last = nd->stack + --nd->depth;
1018	do_delayed_call(&last->done);
1019	if (!(nd->flags & LOOKUP_RCU))
1020		path_put(&last->link);
1021}
1022
1023int sysctl_protected_symlinks __read_mostly = 0;
1024int sysctl_protected_hardlinks __read_mostly = 0;
1025int sysctl_protected_fifos __read_mostly;
1026int sysctl_protected_regular __read_mostly;
1027
1028/**
1029 * may_follow_link - Check symlink following for unsafe situations
1030 * @nd: nameidata pathwalk data
1031 *
1032 * In the case of the sysctl_protected_symlinks sysctl being enabled,
1033 * CAP_DAC_OVERRIDE needs to be specifically ignored if the symlink is
1034 * in a sticky world-writable directory. This is to protect privileged
1035 * processes from failing races against path names that may change out
1036 * from under them by way of other users creating malicious symlinks.
1037 * It will permit symlinks to be followed only when outside a sticky
1038 * world-writable directory, or when the uid of the symlink and follower
1039 * match, or when the directory owner matches the symlink's owner.
1040 *
1041 * Returns 0 if following the symlink is allowed, -ve on error.
1042 */
1043static inline int may_follow_link(struct nameidata *nd, const struct inode *inode)
1044{
1045	struct user_namespace *mnt_userns;
1046	kuid_t i_uid;
1047
1048	if (!sysctl_protected_symlinks)
1049		return 0;
1050
1051	mnt_userns = mnt_user_ns(nd->path.mnt);
1052	i_uid = i_uid_into_mnt(mnt_userns, inode);
1053	/* Allowed if owner and follower match. */
1054	if (uid_eq(current_cred()->fsuid, i_uid))
1055		return 0;
1056
1057	/* Allowed if parent directory not sticky and world-writable. */
1058	if ((nd->dir_mode & (S_ISVTX|S_IWOTH)) != (S_ISVTX|S_IWOTH))
1059		return 0;
1060
1061	/* Allowed if parent directory and link owner match. */
1062	if (uid_valid(nd->dir_uid) && uid_eq(nd->dir_uid, i_uid))
1063		return 0;
1064
1065	if (nd->flags & LOOKUP_RCU)
1066		return -ECHILD;
1067
1068	audit_inode(nd->name, nd->stack[0].link.dentry, 0);
1069	audit_log_path_denied(AUDIT_ANOM_LINK, "follow_link");
1070	return -EACCES;
1071}
1072
1073/**
1074 * safe_hardlink_source - Check for safe hardlink conditions
1075 * @mnt_userns:	user namespace of the mount the inode was found from
1076 * @inode: the source inode to hardlink from
1077 *
1078 * Return false if at least one of the following conditions:
1079 *    - inode is not a regular file
1080 *    - inode is setuid
1081 *    - inode is setgid and group-exec
1082 *    - access failure for read and write
1083 *
1084 * Otherwise returns true.
1085 */
1086static bool safe_hardlink_source(struct user_namespace *mnt_userns,
1087				 struct inode *inode)
1088{
1089	umode_t mode = inode->i_mode;
1090
1091	/* Special files should not get pinned to the filesystem. */
1092	if (!S_ISREG(mode))
1093		return false;
1094
1095	/* Setuid files should not get pinned to the filesystem. */
1096	if (mode & S_ISUID)
1097		return false;
1098
1099	/* Executable setgid files should not get pinned to the filesystem. */
1100	if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP))
1101		return false;
1102
1103	/* Hardlinking to unreadable or unwritable sources is dangerous. */
1104	if (inode_permission(mnt_userns, inode, MAY_READ | MAY_WRITE))
1105		return false;
1106
1107	return true;
1108}
1109
1110/**
1111 * may_linkat - Check permissions for creating a hardlink
1112 * @mnt_userns:	user namespace of the mount the inode was found from
1113 * @link: the source to hardlink from
1114 *
1115 * Block hardlink when all of:
1116 *  - sysctl_protected_hardlinks enabled
1117 *  - fsuid does not match inode
1118 *  - hardlink source is unsafe (see safe_hardlink_source() above)
1119 *  - not CAP_FOWNER in a namespace with the inode owner uid mapped
1120 *
1121 * If the inode has been found through an idmapped mount the user namespace of
1122 * the vfsmount must be passed through @mnt_userns. This function will then take
1123 * care to map the inode according to @mnt_userns before checking permissions.
1124 * On non-idmapped mounts or if permission checking is to be performed on the
1125 * raw inode simply passs init_user_ns.
1126 *
1127 * Returns 0 if successful, -ve on error.
1128 */
1129int may_linkat(struct user_namespace *mnt_userns, struct path *link)
1130{
1131	struct inode *inode = link->dentry->d_inode;
1132
1133	/* Inode writeback is not safe when the uid or gid are invalid. */
1134	if (!uid_valid(i_uid_into_mnt(mnt_userns, inode)) ||
1135	    !gid_valid(i_gid_into_mnt(mnt_userns, inode)))
1136		return -EOVERFLOW;
1137
1138	if (!sysctl_protected_hardlinks)
1139		return 0;
1140
1141	/* Source inode owner (or CAP_FOWNER) can hardlink all they like,
1142	 * otherwise, it must be a safe source.
1143	 */
1144	if (safe_hardlink_source(mnt_userns, inode) ||
1145	    inode_owner_or_capable(mnt_userns, inode))
1146		return 0;
1147
1148	audit_log_path_denied(AUDIT_ANOM_LINK, "linkat");
1149	return -EPERM;
1150}
1151
1152/**
1153 * may_create_in_sticky - Check whether an O_CREAT open in a sticky directory
1154 *			  should be allowed, or not, on files that already
1155 *			  exist.
1156 * @mnt_userns:	user namespace of the mount the inode was found from
1157 * @nd: nameidata pathwalk data
1158 * @inode: the inode of the file to open
1159 *
1160 * Block an O_CREAT open of a FIFO (or a regular file) when:
1161 *   - sysctl_protected_fifos (or sysctl_protected_regular) is enabled
1162 *   - the file already exists
1163 *   - we are in a sticky directory
1164 *   - we don't own the file
1165 *   - the owner of the directory doesn't own the file
1166 *   - the directory is world writable
1167 * If the sysctl_protected_fifos (or sysctl_protected_regular) is set to 2
1168 * the directory doesn't have to be world writable: being group writable will
1169 * be enough.
1170 *
1171 * If the inode has been found through an idmapped mount the user namespace of
1172 * the vfsmount must be passed through @mnt_userns. This function will then take
1173 * care to map the inode according to @mnt_userns before checking permissions.
1174 * On non-idmapped mounts or if permission checking is to be performed on the
1175 * raw inode simply passs init_user_ns.
1176 *
1177 * Returns 0 if the open is allowed, -ve on error.
1178 */
1179static int may_create_in_sticky(struct user_namespace *mnt_userns,
1180				struct nameidata *nd, struct inode *const inode)
1181{
1182	umode_t dir_mode = nd->dir_mode;
1183	kuid_t dir_uid = nd->dir_uid;
1184
1185	if ((!sysctl_protected_fifos && S_ISFIFO(inode->i_mode)) ||
1186	    (!sysctl_protected_regular && S_ISREG(inode->i_mode)) ||
1187	    likely(!(dir_mode & S_ISVTX)) ||
1188	    uid_eq(i_uid_into_mnt(mnt_userns, inode), dir_uid) ||
1189	    uid_eq(current_fsuid(), i_uid_into_mnt(mnt_userns, inode)))
1190		return 0;
1191
1192	if (likely(dir_mode & 0002) ||
1193	    (dir_mode & 0020 &&
1194	     ((sysctl_protected_fifos >= 2 && S_ISFIFO(inode->i_mode)) ||
1195	      (sysctl_protected_regular >= 2 && S_ISREG(inode->i_mode))))) {
1196		const char *operation = S_ISFIFO(inode->i_mode) ?
1197					"sticky_create_fifo" :
1198					"sticky_create_regular";
1199		audit_log_path_denied(AUDIT_ANOM_CREAT, operation);
1200		return -EACCES;
1201	}
1202	return 0;
1203}
1204
1205/*
1206 * follow_up - Find the mountpoint of path's vfsmount
1207 *
1208 * Given a path, find the mountpoint of its source file system.
1209 * Replace @path with the path of the mountpoint in the parent mount.
1210 * Up is towards /.
1211 *
1212 * Return 1 if we went up a level and 0 if we were already at the
1213 * root.
1214 */
1215int follow_up(struct path *path)
1216{
1217	struct mount *mnt = real_mount(path->mnt);
1218	struct mount *parent;
1219	struct dentry *mountpoint;
1220
1221	read_seqlock_excl(&mount_lock);
1222	parent = mnt->mnt_parent;
1223	if (parent == mnt) {
1224		read_sequnlock_excl(&mount_lock);
1225		return 0;
1226	}
1227	mntget(&parent->mnt);
1228	mountpoint = dget(mnt->mnt_mountpoint);
1229	read_sequnlock_excl(&mount_lock);
1230	dput(path->dentry);
1231	path->dentry = mountpoint;
1232	mntput(path->mnt);
1233	path->mnt = &parent->mnt;
1234	return 1;
1235}
1236EXPORT_SYMBOL(follow_up);
1237
1238static bool choose_mountpoint_rcu(struct mount *m, const struct path *root,
1239				  struct path *path, unsigned *seqp)
1240{
1241	while (mnt_has_parent(m)) {
1242		struct dentry *mountpoint = m->mnt_mountpoint;
1243
1244		m = m->mnt_parent;
1245		if (unlikely(root->dentry == mountpoint &&
1246			     root->mnt == &m->mnt))
1247			break;
1248		if (mountpoint != m->mnt.mnt_root) {
1249			path->mnt = &m->mnt;
1250			path->dentry = mountpoint;
1251			*seqp = read_seqcount_begin(&mountpoint->d_seq);
1252			return true;
1253		}
1254	}
1255	return false;
1256}
1257
1258static bool choose_mountpoint(struct mount *m, const struct path *root,
1259			      struct path *path)
1260{
1261	bool found;
1262
1263	rcu_read_lock();
1264	while (1) {
1265		unsigned seq, mseq = read_seqbegin(&mount_lock);
1266
1267		found = choose_mountpoint_rcu(m, root, path, &seq);
1268		if (unlikely(!found)) {
1269			if (!read_seqretry(&mount_lock, mseq))
1270				break;
1271		} else {
1272			if (likely(__legitimize_path(path, seq, mseq)))
1273				break;
1274			rcu_read_unlock();
1275			path_put(path);
1276			rcu_read_lock();
1277		}
1278	}
1279	rcu_read_unlock();
1280	return found;
1281}
1282
1283/*
1284 * Perform an automount
1285 * - return -EISDIR to tell follow_managed() to stop and return the path we
1286 *   were called with.
1287 */
1288static int follow_automount(struct path *path, int *count, unsigned lookup_flags)
1289{
1290	struct dentry *dentry = path->dentry;
1291
1292	/* We don't want to mount if someone's just doing a stat -
1293	 * unless they're stat'ing a directory and appended a '/' to
1294	 * the name.
1295	 *
1296	 * We do, however, want to mount if someone wants to open or
1297	 * create a file of any type under the mountpoint, wants to
1298	 * traverse through the mountpoint or wants to open the
1299	 * mounted directory.  Also, autofs may mark negative dentries
1300	 * as being automount points.  These will need the attentions
1301	 * of the daemon to instantiate them before they can be used.
1302	 */
1303	if (!(lookup_flags & (LOOKUP_PARENT | LOOKUP_DIRECTORY |
1304			   LOOKUP_OPEN | LOOKUP_CREATE | LOOKUP_AUTOMOUNT)) &&
1305	    dentry->d_inode)
1306		return -EISDIR;
1307
1308	if (count && (*count)++ >= MAXSYMLINKS)
1309		return -ELOOP;
1310
1311	return finish_automount(dentry->d_op->d_automount(path), path);
1312}
1313
1314/*
1315 * mount traversal - out-of-line part.  One note on ->d_flags accesses -
1316 * dentries are pinned but not locked here, so negative dentry can go
1317 * positive right under us.  Use of smp_load_acquire() provides a barrier
1318 * sufficient for ->d_inode and ->d_flags consistency.
1319 */
1320static int __traverse_mounts(struct path *path, unsigned flags, bool *jumped,
1321			     int *count, unsigned lookup_flags)
1322{
1323	struct vfsmount *mnt = path->mnt;
1324	bool need_mntput = false;
1325	int ret = 0;
1326
1327	while (flags & DCACHE_MANAGED_DENTRY) {
1328		/* Allow the filesystem to manage the transit without i_mutex
1329		 * being held. */
1330		if (flags & DCACHE_MANAGE_TRANSIT) {
1331			ret = path->dentry->d_op->d_manage(path, false);
1332			flags = smp_load_acquire(&path->dentry->d_flags);
1333			if (ret < 0)
1334				break;
1335		}
1336
1337		if (flags & DCACHE_MOUNTED) {	// something's mounted on it..
1338			struct vfsmount *mounted = lookup_mnt(path);
1339			if (mounted) {		// ... in our namespace
1340				dput(path->dentry);
1341				if (need_mntput)
1342					mntput(path->mnt);
1343				path->mnt = mounted;
1344				path->dentry = dget(mounted->mnt_root);
1345				// here we know it's positive
1346				flags = path->dentry->d_flags;
1347				need_mntput = true;
1348				continue;
1349			}
1350		}
1351
1352		if (!(flags & DCACHE_NEED_AUTOMOUNT))
1353			break;
1354
1355		// uncovered automount point
1356		ret = follow_automount(path, count, lookup_flags);
1357		flags = smp_load_acquire(&path->dentry->d_flags);
1358		if (ret < 0)
1359			break;
1360	}
1361
1362	if (ret == -EISDIR)
1363		ret = 0;
1364	// possible if you race with several mount --move
1365	if (need_mntput && path->mnt == mnt)
1366		mntput(path->mnt);
1367	if (!ret && unlikely(d_flags_negative(flags)))
1368		ret = -ENOENT;
1369	*jumped = need_mntput;
1370	return ret;
1371}
1372
1373static inline int traverse_mounts(struct path *path, bool *jumped,
1374				  int *count, unsigned lookup_flags)
1375{
1376	unsigned flags = smp_load_acquire(&path->dentry->d_flags);
1377
1378	/* fastpath */
1379	if (likely(!(flags & DCACHE_MANAGED_DENTRY))) {
1380		*jumped = false;
1381		if (unlikely(d_flags_negative(flags)))
1382			return -ENOENT;
1383		return 0;
1384	}
1385	return __traverse_mounts(path, flags, jumped, count, lookup_flags);
1386}
1387
1388int follow_down_one(struct path *path)
1389{
1390	struct vfsmount *mounted;
1391
1392	mounted = lookup_mnt(path);
1393	if (mounted) {
1394		dput(path->dentry);
1395		mntput(path->mnt);
1396		path->mnt = mounted;
1397		path->dentry = dget(mounted->mnt_root);
1398		return 1;
1399	}
1400	return 0;
1401}
1402EXPORT_SYMBOL(follow_down_one);
1403
1404/*
1405 * Follow down to the covering mount currently visible to userspace.  At each
1406 * point, the filesystem owning that dentry may be queried as to whether the
1407 * caller is permitted to proceed or not.
1408 */
1409int follow_down(struct path *path)
1410{
1411	struct vfsmount *mnt = path->mnt;
1412	bool jumped;
1413	int ret = traverse_mounts(path, &jumped, NULL, 0);
1414
1415	if (path->mnt != mnt)
1416		mntput(mnt);
1417	return ret;
1418}
1419EXPORT_SYMBOL(follow_down);
1420
1421/*
1422 * Try to skip to top of mountpoint pile in rcuwalk mode.  Fail if
1423 * we meet a managed dentry that would need blocking.
1424 */
1425static bool __follow_mount_rcu(struct nameidata *nd, struct path *path,
1426			       struct inode **inode, unsigned *seqp)
1427{
1428	struct dentry *dentry = path->dentry;
1429	unsigned int flags = dentry->d_flags;
1430
1431	if (likely(!(flags & DCACHE_MANAGED_DENTRY)))
1432		return true;
1433
1434	if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1435		return false;
1436
1437	for (;;) {
1438		/*
1439		 * Don't forget we might have a non-mountpoint managed dentry
1440		 * that wants to block transit.
1441		 */
1442		if (unlikely(flags & DCACHE_MANAGE_TRANSIT)) {
1443			int res = dentry->d_op->d_manage(path, true);
1444			if (res)
1445				return res == -EISDIR;
1446			flags = dentry->d_flags;
1447		}
1448
1449		if (flags & DCACHE_MOUNTED) {
1450			struct mount *mounted = __lookup_mnt(path->mnt, dentry);
1451			if (mounted) {
1452				path->mnt = &mounted->mnt;
1453				dentry = path->dentry = mounted->mnt.mnt_root;
1454				nd->state |= ND_JUMPED;
1455				*seqp = read_seqcount_begin(&dentry->d_seq);
1456				*inode = dentry->d_inode;
1457				/*
1458				 * We don't need to re-check ->d_seq after this
1459				 * ->d_inode read - there will be an RCU delay
1460				 * between mount hash removal and ->mnt_root
1461				 * becoming unpinned.
1462				 */
1463				flags = dentry->d_flags;
1464				continue;
1465			}
1466			if (read_seqretry(&mount_lock, nd->m_seq))
1467				return false;
1468		}
1469		return !(flags & DCACHE_NEED_AUTOMOUNT);
1470	}
1471}
1472
1473static inline int handle_mounts(struct nameidata *nd, struct dentry *dentry,
1474			  struct path *path, struct inode **inode,
1475			  unsigned int *seqp)
1476{
1477	bool jumped;
1478	int ret;
1479
1480	path->mnt = nd->path.mnt;
1481	path->dentry = dentry;
1482	if (nd->flags & LOOKUP_RCU) {
1483		unsigned int seq = *seqp;
1484		if (unlikely(!*inode))
1485			return -ENOENT;
1486		if (likely(__follow_mount_rcu(nd, path, inode, seqp)))
1487			return 0;
1488		if (!try_to_unlazy_next(nd, dentry, seq))
1489			return -ECHILD;
1490		// *path might've been clobbered by __follow_mount_rcu()
1491		path->mnt = nd->path.mnt;
1492		path->dentry = dentry;
1493	}
1494	ret = traverse_mounts(path, &jumped, &nd->total_link_count, nd->flags);
1495	if (jumped) {
1496		if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1497			ret = -EXDEV;
1498		else
1499			nd->state |= ND_JUMPED;
1500	}
1501	if (unlikely(ret)) {
1502		dput(path->dentry);
1503		if (path->mnt != nd->path.mnt)
1504			mntput(path->mnt);
1505	} else {
1506		*inode = d_backing_inode(path->dentry);
1507		*seqp = 0; /* out of RCU mode, so the value doesn't matter */
1508	}
1509	return ret;
1510}
1511
1512/*
1513 * This looks up the name in dcache and possibly revalidates the found dentry.
1514 * NULL is returned if the dentry does not exist in the cache.
1515 */
1516static struct dentry *lookup_dcache(const struct qstr *name,
1517				    struct dentry *dir,
1518				    unsigned int flags)
1519{
1520	struct dentry *dentry = d_lookup(dir, name);
1521	if (dentry) {
1522		int error = d_revalidate(dentry, flags);
1523		if (unlikely(error <= 0)) {
1524			if (!error)
1525				d_invalidate(dentry);
1526			dput(dentry);
1527			return ERR_PTR(error);
1528		}
1529	}
1530	return dentry;
1531}
1532
1533/*
1534 * Parent directory has inode locked exclusive.  This is one
1535 * and only case when ->lookup() gets called on non in-lookup
1536 * dentries - as the matter of fact, this only gets called
1537 * when directory is guaranteed to have no in-lookup children
1538 * at all.
1539 */
1540static struct dentry *__lookup_hash(const struct qstr *name,
1541		struct dentry *base, unsigned int flags)
1542{
1543	struct dentry *dentry = lookup_dcache(name, base, flags);
1544	struct dentry *old;
1545	struct inode *dir = base->d_inode;
1546
1547	if (dentry)
1548		return dentry;
1549
1550	/* Don't create child dentry for a dead directory. */
1551	if (unlikely(IS_DEADDIR(dir)))
1552		return ERR_PTR(-ENOENT);
1553
1554	dentry = d_alloc(base, name);
1555	if (unlikely(!dentry))
1556		return ERR_PTR(-ENOMEM);
1557
1558	old = dir->i_op->lookup(dir, dentry, flags);
1559	if (unlikely(old)) {
1560		dput(dentry);
1561		dentry = old;
1562	}
1563	return dentry;
1564}
1565
1566static struct dentry *lookup_fast(struct nameidata *nd,
1567				  struct inode **inode,
1568			          unsigned *seqp)
1569{
1570	struct dentry *dentry, *parent = nd->path.dentry;
1571	int status = 1;
1572
1573	/*
1574	 * Rename seqlock is not required here because in the off chance
1575	 * of a false negative due to a concurrent rename, the caller is
1576	 * going to fall back to non-racy lookup.
1577	 */
1578	if (nd->flags & LOOKUP_RCU) {
1579		unsigned seq;
1580		dentry = __d_lookup_rcu(parent, &nd->last, &seq);
1581		if (unlikely(!dentry)) {
1582			if (!try_to_unlazy(nd))
1583				return ERR_PTR(-ECHILD);
1584			return NULL;
1585		}
1586
1587		/*
1588		 * This sequence count validates that the inode matches
1589		 * the dentry name information from lookup.
1590		 */
1591		*inode = d_backing_inode(dentry);
1592		if (unlikely(read_seqcount_retry(&dentry->d_seq, seq)))
1593			return ERR_PTR(-ECHILD);
1594
1595		/*
1596		 * This sequence count validates that the parent had no
1597		 * changes while we did the lookup of the dentry above.
1598		 *
1599		 * The memory barrier in read_seqcount_begin of child is
1600		 *  enough, we can use __read_seqcount_retry here.
1601		 */
1602		if (unlikely(__read_seqcount_retry(&parent->d_seq, nd->seq)))
1603			return ERR_PTR(-ECHILD);
1604
1605		*seqp = seq;
1606		status = d_revalidate(dentry, nd->flags);
1607		if (likely(status > 0))
1608			return dentry;
1609		if (!try_to_unlazy_next(nd, dentry, seq))
1610			return ERR_PTR(-ECHILD);
1611		if (status == -ECHILD)
1612			/* we'd been told to redo it in non-rcu mode */
1613			status = d_revalidate(dentry, nd->flags);
1614	} else {
1615		dentry = __d_lookup(parent, &nd->last);
1616		if (unlikely(!dentry))
1617			return NULL;
1618		status = d_revalidate(dentry, nd->flags);
1619	}
1620	if (unlikely(status <= 0)) {
1621		if (!status)
1622			d_invalidate(dentry);
1623		dput(dentry);
1624		return ERR_PTR(status);
1625	}
1626	return dentry;
1627}
1628
1629/* Fast lookup failed, do it the slow way */
1630static struct dentry *__lookup_slow(const struct qstr *name,
1631				    struct dentry *dir,
1632				    unsigned int flags)
1633{
1634	struct dentry *dentry, *old;
1635	struct inode *inode = dir->d_inode;
1636	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
1637
1638	/* Don't go there if it's already dead */
1639	if (unlikely(IS_DEADDIR(inode)))
1640		return ERR_PTR(-ENOENT);
1641again:
1642	dentry = d_alloc_parallel(dir, name, &wq);
1643	if (IS_ERR(dentry))
1644		return dentry;
1645	if (unlikely(!d_in_lookup(dentry))) {
1646		int error = d_revalidate(dentry, flags);
1647		if (unlikely(error <= 0)) {
1648			if (!error) {
1649				d_invalidate(dentry);
1650				dput(dentry);
1651				goto again;
1652			}
1653			dput(dentry);
1654			dentry = ERR_PTR(error);
1655		}
1656	} else {
1657		old = inode->i_op->lookup(inode, dentry, flags);
1658		d_lookup_done(dentry);
1659		if (unlikely(old)) {
1660			dput(dentry);
1661			dentry = old;
1662		}
1663	}
1664	return dentry;
1665}
1666
1667static struct dentry *lookup_slow(const struct qstr *name,
1668				  struct dentry *dir,
1669				  unsigned int flags)
1670{
1671	struct inode *inode = dir->d_inode;
1672	struct dentry *res;
1673	inode_lock_shared(inode);
1674	res = __lookup_slow(name, dir, flags);
1675	inode_unlock_shared(inode);
1676	return res;
1677}
1678
1679static inline int may_lookup(struct user_namespace *mnt_userns,
1680			     struct nameidata *nd)
1681{
1682	if (nd->flags & LOOKUP_RCU) {
1683		int err = inode_permission(mnt_userns, nd->inode, MAY_EXEC|MAY_NOT_BLOCK);
1684		if (err != -ECHILD || !try_to_unlazy(nd))
1685			return err;
1686	}
1687	return inode_permission(mnt_userns, nd->inode, MAY_EXEC);
1688}
1689
1690static int reserve_stack(struct nameidata *nd, struct path *link, unsigned seq)
1691{
1692	if (unlikely(nd->total_link_count++ >= MAXSYMLINKS))
1693		return -ELOOP;
1694
1695	if (likely(nd->depth != EMBEDDED_LEVELS))
1696		return 0;
1697	if (likely(nd->stack != nd->internal))
1698		return 0;
1699	if (likely(nd_alloc_stack(nd)))
1700		return 0;
1701
1702	if (nd->flags & LOOKUP_RCU) {
1703		// we need to grab link before we do unlazy.  And we can't skip
1704		// unlazy even if we fail to grab the link - cleanup needs it
1705		bool grabbed_link = legitimize_path(nd, link, seq);
1706
1707		if (!try_to_unlazy(nd) != 0 || !grabbed_link)
1708			return -ECHILD;
1709
1710		if (nd_alloc_stack(nd))
1711			return 0;
1712	}
1713	return -ENOMEM;
1714}
1715
1716enum {WALK_TRAILING = 1, WALK_MORE = 2, WALK_NOFOLLOW = 4};
1717
1718static const char *pick_link(struct nameidata *nd, struct path *link,
1719		     struct inode *inode, unsigned seq, int flags)
1720{
1721	struct saved *last;
1722	const char *res;
1723	int error = reserve_stack(nd, link, seq);
1724
1725	if (unlikely(error)) {
1726		if (!(nd->flags & LOOKUP_RCU))
1727			path_put(link);
1728		return ERR_PTR(error);
1729	}
1730	last = nd->stack + nd->depth++;
1731	last->link = *link;
1732	clear_delayed_call(&last->done);
1733	last->seq = seq;
1734
1735	if (flags & WALK_TRAILING) {
1736		error = may_follow_link(nd, inode);
1737		if (unlikely(error))
1738			return ERR_PTR(error);
1739	}
1740
1741	if (unlikely(nd->flags & LOOKUP_NO_SYMLINKS) ||
1742			unlikely(link->mnt->mnt_flags & MNT_NOSYMFOLLOW))
1743		return ERR_PTR(-ELOOP);
1744
1745	if (!(nd->flags & LOOKUP_RCU)) {
1746		touch_atime(&last->link);
1747		cond_resched();
1748	} else if (atime_needs_update(&last->link, inode)) {
1749		if (!try_to_unlazy(nd))
1750			return ERR_PTR(-ECHILD);
1751		touch_atime(&last->link);
1752	}
1753
1754	error = security_inode_follow_link(link->dentry, inode,
1755					   nd->flags & LOOKUP_RCU);
1756	if (unlikely(error))
1757		return ERR_PTR(error);
1758
1759	res = READ_ONCE(inode->i_link);
1760	if (!res) {
1761		const char * (*get)(struct dentry *, struct inode *,
1762				struct delayed_call *);
1763		get = inode->i_op->get_link;
1764		if (nd->flags & LOOKUP_RCU) {
1765			res = get(NULL, inode, &last->done);
1766			if (res == ERR_PTR(-ECHILD) && try_to_unlazy(nd))
1767				res = get(link->dentry, inode, &last->done);
1768		} else {
1769			res = get(link->dentry, inode, &last->done);
1770		}
1771		if (!res)
1772			goto all_done;
1773		if (IS_ERR(res))
1774			return res;
1775	}
1776	if (*res == '/') {
1777		error = nd_jump_root(nd);
1778		if (unlikely(error))
1779			return ERR_PTR(error);
1780		while (unlikely(*++res == '/'))
1781			;
1782	}
1783	if (*res)
1784		return res;
1785all_done: // pure jump
1786	put_link(nd);
1787	return NULL;
1788}
1789
1790/*
1791 * Do we need to follow links? We _really_ want to be able
1792 * to do this check without having to look at inode->i_op,
1793 * so we keep a cache of "no, this doesn't need follow_link"
1794 * for the common case.
1795 */
1796static const char *step_into(struct nameidata *nd, int flags,
1797		     struct dentry *dentry, struct inode *inode, unsigned seq)
1798{
1799	struct path path;
1800	int err = handle_mounts(nd, dentry, &path, &inode, &seq);
1801
1802	if (err < 0)
1803		return ERR_PTR(err);
1804	if (likely(!d_is_symlink(path.dentry)) ||
1805	   ((flags & WALK_TRAILING) && !(nd->flags & LOOKUP_FOLLOW)) ||
1806	   (flags & WALK_NOFOLLOW)) {
1807		/* not a symlink or should not follow */
1808		if (!(nd->flags & LOOKUP_RCU)) {
1809			dput(nd->path.dentry);
1810			if (nd->path.mnt != path.mnt)
1811				mntput(nd->path.mnt);
1812		}
1813		nd->path = path;
1814		nd->inode = inode;
1815		nd->seq = seq;
1816		return NULL;
1817	}
1818	if (nd->flags & LOOKUP_RCU) {
1819		/* make sure that d_is_symlink above matches inode */
1820		if (read_seqcount_retry(&path.dentry->d_seq, seq))
1821			return ERR_PTR(-ECHILD);
1822	} else {
1823		if (path.mnt == nd->path.mnt)
1824			mntget(path.mnt);
1825	}
1826	return pick_link(nd, &path, inode, seq, flags);
1827}
1828
1829static struct dentry *follow_dotdot_rcu(struct nameidata *nd,
1830					struct inode **inodep,
1831					unsigned *seqp)
1832{
1833	struct dentry *parent, *old;
1834
1835	if (path_equal(&nd->path, &nd->root))
1836		goto in_root;
1837	if (unlikely(nd->path.dentry == nd->path.mnt->mnt_root)) {
1838		struct path path;
1839		unsigned seq;
1840		if (!choose_mountpoint_rcu(real_mount(nd->path.mnt),
1841					   &nd->root, &path, &seq))
1842			goto in_root;
1843		if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1844			return ERR_PTR(-ECHILD);
1845		nd->path = path;
1846		nd->inode = path.dentry->d_inode;
1847		nd->seq = seq;
1848		if (unlikely(read_seqretry(&mount_lock, nd->m_seq)))
1849			return ERR_PTR(-ECHILD);
1850		/* we know that mountpoint was pinned */
1851	}
1852	old = nd->path.dentry;
1853	parent = old->d_parent;
1854	*inodep = parent->d_inode;
1855	*seqp = read_seqcount_begin(&parent->d_seq);
1856	if (unlikely(read_seqcount_retry(&old->d_seq, nd->seq)))
1857		return ERR_PTR(-ECHILD);
1858	if (unlikely(!path_connected(nd->path.mnt, parent)))
1859		return ERR_PTR(-ECHILD);
1860	return parent;
1861in_root:
1862	if (unlikely(read_seqretry(&mount_lock, nd->m_seq)))
1863		return ERR_PTR(-ECHILD);
1864	if (unlikely(nd->flags & LOOKUP_BENEATH))
1865		return ERR_PTR(-ECHILD);
1866	return NULL;
1867}
1868
1869static struct dentry *follow_dotdot(struct nameidata *nd,
1870				 struct inode **inodep,
1871				 unsigned *seqp)
1872{
1873	struct dentry *parent;
1874
1875	if (path_equal(&nd->path, &nd->root))
1876		goto in_root;
1877	if (unlikely(nd->path.dentry == nd->path.mnt->mnt_root)) {
1878		struct path path;
1879
1880		if (!choose_mountpoint(real_mount(nd->path.mnt),
1881				       &nd->root, &path))
1882			goto in_root;
1883		path_put(&nd->path);
1884		nd->path = path;
1885		nd->inode = path.dentry->d_inode;
1886		if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1887			return ERR_PTR(-EXDEV);
1888	}
1889	/* rare case of legitimate dget_parent()... */
1890	parent = dget_parent(nd->path.dentry);
1891	if (unlikely(!path_connected(nd->path.mnt, parent))) {
1892		dput(parent);
1893		return ERR_PTR(-ENOENT);
1894	}
1895	*seqp = 0;
1896	*inodep = parent->d_inode;
1897	return parent;
1898
1899in_root:
1900	if (unlikely(nd->flags & LOOKUP_BENEATH))
1901		return ERR_PTR(-EXDEV);
1902	dget(nd->path.dentry);
1903	return NULL;
1904}
1905
1906static const char *handle_dots(struct nameidata *nd, int type)
1907{
1908	if (type == LAST_DOTDOT) {
1909		const char *error = NULL;
1910		struct dentry *parent;
1911		struct inode *inode;
1912		unsigned seq;
1913
1914		if (!nd->root.mnt) {
1915			error = ERR_PTR(set_root(nd));
1916			if (error)
1917				return error;
1918		}
1919		if (nd->flags & LOOKUP_RCU)
1920			parent = follow_dotdot_rcu(nd, &inode, &seq);
1921		else
1922			parent = follow_dotdot(nd, &inode, &seq);
1923		if (IS_ERR(parent))
1924			return ERR_CAST(parent);
1925		if (unlikely(!parent))
1926			error = step_into(nd, WALK_NOFOLLOW,
1927					 nd->path.dentry, nd->inode, nd->seq);
1928		else
1929			error = step_into(nd, WALK_NOFOLLOW,
1930					 parent, inode, seq);
1931		if (unlikely(error))
1932			return error;
1933
1934		if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) {
1935			/*
1936			 * If there was a racing rename or mount along our
1937			 * path, then we can't be sure that ".." hasn't jumped
1938			 * above nd->root (and so userspace should retry or use
1939			 * some fallback).
1940			 */
1941			smp_rmb();
1942			if (unlikely(__read_seqcount_retry(&mount_lock.seqcount, nd->m_seq)))
1943				return ERR_PTR(-EAGAIN);
1944			if (unlikely(__read_seqcount_retry(&rename_lock.seqcount, nd->r_seq)))
1945				return ERR_PTR(-EAGAIN);
1946		}
1947	}
1948	return NULL;
1949}
1950
1951static const char *walk_component(struct nameidata *nd, int flags)
1952{
1953	struct dentry *dentry;
1954	struct inode *inode;
1955	unsigned seq;
1956	/*
1957	 * "." and ".." are special - ".." especially so because it has
1958	 * to be able to know about the current root directory and
1959	 * parent relationships.
1960	 */
1961	if (unlikely(nd->last_type != LAST_NORM)) {
1962		if (!(flags & WALK_MORE) && nd->depth)
1963			put_link(nd);
1964		return handle_dots(nd, nd->last_type);
1965	}
1966	dentry = lookup_fast(nd, &inode, &seq);
1967	if (IS_ERR(dentry))
1968		return ERR_CAST(dentry);
1969	if (unlikely(!dentry)) {
1970		dentry = lookup_slow(&nd->last, nd->path.dentry, nd->flags);
1971		if (IS_ERR(dentry))
1972			return ERR_CAST(dentry);
1973	}
1974	if (!(flags & WALK_MORE) && nd->depth)
1975		put_link(nd);
1976	return step_into(nd, flags, dentry, inode, seq);
1977}
1978
1979/*
1980 * We can do the critical dentry name comparison and hashing
1981 * operations one word at a time, but we are limited to:
1982 *
1983 * - Architectures with fast unaligned word accesses. We could
1984 *   do a "get_unaligned()" if this helps and is sufficiently
1985 *   fast.
1986 *
1987 * - non-CONFIG_DEBUG_PAGEALLOC configurations (so that we
1988 *   do not trap on the (extremely unlikely) case of a page
1989 *   crossing operation.
1990 *
1991 * - Furthermore, we need an efficient 64-bit compile for the
1992 *   64-bit case in order to generate the "number of bytes in
1993 *   the final mask". Again, that could be replaced with a
1994 *   efficient population count instruction or similar.
1995 */
1996#ifdef CONFIG_DCACHE_WORD_ACCESS
1997
1998#include <asm/word-at-a-time.h>
1999
2000#ifdef HASH_MIX
2001
2002/* Architecture provides HASH_MIX and fold_hash() in <asm/hash.h> */
2003
2004#elif defined(CONFIG_64BIT)
2005/*
2006 * Register pressure in the mixing function is an issue, particularly
2007 * on 32-bit x86, but almost any function requires one state value and
2008 * one temporary.  Instead, use a function designed for two state values
2009 * and no temporaries.
2010 *
2011 * This function cannot create a collision in only two iterations, so
2012 * we have two iterations to achieve avalanche.  In those two iterations,
2013 * we have six layers of mixing, which is enough to spread one bit's
2014 * influence out to 2^6 = 64 state bits.
2015 *
2016 * Rotate constants are scored by considering either 64 one-bit input
2017 * deltas or 64*63/2 = 2016 two-bit input deltas, and finding the
2018 * probability of that delta causing a change to each of the 128 output
2019 * bits, using a sample of random initial states.
2020 *
2021 * The Shannon entropy of the computed probabilities is then summed
2022 * to produce a score.  Ideally, any input change has a 50% chance of
2023 * toggling any given output bit.
2024 *
2025 * Mixing scores (in bits) for (12,45):
2026 * Input delta: 1-bit      2-bit
2027 * 1 round:     713.3    42542.6
2028 * 2 rounds:   2753.7   140389.8
2029 * 3 rounds:   5954.1   233458.2
2030 * 4 rounds:   7862.6   256672.2
2031 * Perfect:    8192     258048
2032 *            (64*128) (64*63/2 * 128)
2033 */
2034#define HASH_MIX(x, y, a)	\
2035	(	x ^= (a),	\
2036	y ^= x,	x = rol64(x,12),\
2037	x += y,	y = rol64(y,45),\
2038	y *= 9			)
2039
2040/*
2041 * Fold two longs into one 32-bit hash value.  This must be fast, but
2042 * latency isn't quite as critical, as there is a fair bit of additional
2043 * work done before the hash value is used.
2044 */
2045static inline unsigned int fold_hash(unsigned long x, unsigned long y)
2046{
2047	y ^= x * GOLDEN_RATIO_64;
2048	y *= GOLDEN_RATIO_64;
2049	return y >> 32;
2050}
2051
2052#else	/* 32-bit case */
2053
2054/*
2055 * Mixing scores (in bits) for (7,20):
2056 * Input delta: 1-bit      2-bit
2057 * 1 round:     330.3     9201.6
2058 * 2 rounds:   1246.4    25475.4
2059 * 3 rounds:   1907.1    31295.1
2060 * 4 rounds:   2042.3    31718.6
2061 * Perfect:    2048      31744
2062 *            (32*64)   (32*31/2 * 64)
2063 */
2064#define HASH_MIX(x, y, a)	\
2065	(	x ^= (a),	\
2066	y ^= x,	x = rol32(x, 7),\
2067	x += y,	y = rol32(y,20),\
2068	y *= 9			)
2069
2070static inline unsigned int fold_hash(unsigned long x, unsigned long y)
2071{
2072	/* Use arch-optimized multiply if one exists */
2073	return __hash_32(y ^ __hash_32(x));
2074}
2075
2076#endif
2077
2078/*
2079 * Return the hash of a string of known length.  This is carfully
2080 * designed to match hash_name(), which is the more critical function.
2081 * In particular, we must end by hashing a final word containing 0..7
2082 * payload bytes, to match the way that hash_name() iterates until it
2083 * finds the delimiter after the name.
2084 */
2085unsigned int full_name_hash(const void *salt, const char *name, unsigned int len)
2086{
2087	unsigned long a, x = 0, y = (unsigned long)salt;
2088
2089	for (;;) {
2090		if (!len)
2091			goto done;
2092		a = load_unaligned_zeropad(name);
2093		if (len < sizeof(unsigned long))
2094			break;
2095		HASH_MIX(x, y, a);
2096		name += sizeof(unsigned long);
2097		len -= sizeof(unsigned long);
2098	}
2099	x ^= a & bytemask_from_count(len);
2100done:
2101	return fold_hash(x, y);
2102}
2103EXPORT_SYMBOL(full_name_hash);
2104
2105/* Return the "hash_len" (hash and length) of a null-terminated string */
2106u64 hashlen_string(const void *salt, const char *name)
2107{
2108	unsigned long a = 0, x = 0, y = (unsigned long)salt;
2109	unsigned long adata, mask, len;
2110	const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
2111
2112	len = 0;
2113	goto inside;
2114
2115	do {
2116		HASH_MIX(x, y, a);
2117		len += sizeof(unsigned long);
2118inside:
2119		a = load_unaligned_zeropad(name+len);
2120	} while (!has_zero(a, &adata, &constants));
2121
2122	adata = prep_zero_mask(a, adata, &constants);
2123	mask = create_zero_mask(adata);
2124	x ^= a & zero_bytemask(mask);
2125
2126	return hashlen_create(fold_hash(x, y), len + find_zero(mask));
2127}
2128EXPORT_SYMBOL(hashlen_string);
2129
2130/*
2131 * Calculate the length and hash of the path component, and
2132 * return the "hash_len" as the result.
2133 */
2134static inline u64 hash_name(const void *salt, const char *name)
2135{
2136	unsigned long a = 0, b, x = 0, y = (unsigned long)salt;
2137	unsigned long adata, bdata, mask, len;
2138	const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
2139
2140	len = 0;
2141	goto inside;
2142
2143	do {
2144		HASH_MIX(x, y, a);
2145		len += sizeof(unsigned long);
2146inside:
2147		a = load_unaligned_zeropad(name+len);
2148		b = a ^ REPEAT_BYTE('/');
2149	} while (!(has_zero(a, &adata, &constants) | has_zero(b, &bdata, &constants)));
2150
2151	adata = prep_zero_mask(a, adata, &constants);
2152	bdata = prep_zero_mask(b, bdata, &constants);
2153	mask = create_zero_mask(adata | bdata);
2154	x ^= a & zero_bytemask(mask);
2155
2156	return hashlen_create(fold_hash(x, y), len + find_zero(mask));
2157}
2158
2159#else	/* !CONFIG_DCACHE_WORD_ACCESS: Slow, byte-at-a-time version */
2160
2161/* Return the hash of a string of known length */
2162unsigned int full_name_hash(const void *salt, const char *name, unsigned int len)
2163{
2164	unsigned long hash = init_name_hash(salt);
2165	while (len--)
2166		hash = partial_name_hash((unsigned char)*name++, hash);
2167	return end_name_hash(hash);
2168}
2169EXPORT_SYMBOL(full_name_hash);
2170
2171/* Return the "hash_len" (hash and length) of a null-terminated string */
2172u64 hashlen_string(const void *salt, const char *name)
2173{
2174	unsigned long hash = init_name_hash(salt);
2175	unsigned long len = 0, c;
2176
2177	c = (unsigned char)*name;
2178	while (c) {
2179		len++;
2180		hash = partial_name_hash(c, hash);
2181		c = (unsigned char)name[len];
2182	}
2183	return hashlen_create(end_name_hash(hash), len);
2184}
2185EXPORT_SYMBOL(hashlen_string);
2186
2187/*
2188 * We know there's a real path component here of at least
2189 * one character.
2190 */
2191static inline u64 hash_name(const void *salt, const char *name)
2192{
2193	unsigned long hash = init_name_hash(salt);
2194	unsigned long len = 0, c;
2195
2196	c = (unsigned char)*name;
2197	do {
2198		len++;
2199		hash = partial_name_hash(c, hash);
2200		c = (unsigned char)name[len];
2201	} while (c && c != '/');
2202	return hashlen_create(end_name_hash(hash), len);
2203}
2204
2205#endif
2206
2207/*
2208 * Name resolution.
2209 * This is the basic name resolution function, turning a pathname into
2210 * the final dentry. We expect 'base' to be positive and a directory.
2211 *
2212 * Returns 0 and nd will have valid dentry and mnt on success.
2213 * Returns error and drops reference to input namei data on failure.
2214 */
2215static int link_path_walk(const char *name, struct nameidata *nd)
2216{
2217	int depth = 0; // depth <= nd->depth
2218	int err;
2219
2220	nd->last_type = LAST_ROOT;
2221	nd->flags |= LOOKUP_PARENT;
2222	if (IS_ERR(name))
2223		return PTR_ERR(name);
2224	while (*name=='/')
2225		name++;
2226	if (!*name) {
2227		nd->dir_mode = 0; // short-circuit the 'hardening' idiocy
2228		return 0;
2229	}
2230
2231	/* At this point we know we have a real path component. */
2232	for(;;) {
2233		struct user_namespace *mnt_userns;
2234		const char *link;
2235		u64 hash_len;
2236		int type;
2237
2238		mnt_userns = mnt_user_ns(nd->path.mnt);
2239		err = may_lookup(mnt_userns, nd);
2240		if (err)
2241			return err;
2242
2243		hash_len = hash_name(nd->path.dentry, name);
2244
2245		type = LAST_NORM;
2246		if (name[0] == '.') switch (hashlen_len(hash_len)) {
2247			case 2:
2248				if (name[1] == '.') {
2249					type = LAST_DOTDOT;
2250					nd->state |= ND_JUMPED;
2251				}
2252				break;
2253			case 1:
2254				type = LAST_DOT;
2255		}
2256		if (likely(type == LAST_NORM)) {
2257			struct dentry *parent = nd->path.dentry;
2258			nd->state &= ~ND_JUMPED;
2259			if (unlikely(parent->d_flags & DCACHE_OP_HASH)) {
2260				struct qstr this = { { .hash_len = hash_len }, .name = name };
2261				err = parent->d_op->d_hash(parent, &this);
2262				if (err < 0)
2263					return err;
2264				hash_len = this.hash_len;
2265				name = this.name;
2266			}
2267		}
2268
2269		nd->last.hash_len = hash_len;
2270		nd->last.name = name;
2271		nd->last_type = type;
2272
2273		name += hashlen_len(hash_len);
2274		if (!*name)
2275			goto OK;
2276		/*
2277		 * If it wasn't NUL, we know it was '/'. Skip that
2278		 * slash, and continue until no more slashes.
2279		 */
2280		do {
2281			name++;
2282		} while (unlikely(*name == '/'));
2283		if (unlikely(!*name)) {
2284OK:
2285			/* pathname or trailing symlink, done */
2286			if (!depth) {
2287				nd->dir_uid = i_uid_into_mnt(mnt_userns, nd->inode);
2288				nd->dir_mode = nd->inode->i_mode;
2289				nd->flags &= ~LOOKUP_PARENT;
2290				return 0;
2291			}
2292			/* last component of nested symlink */
2293			name = nd->stack[--depth].name;
2294			link = walk_component(nd, 0);
2295		} else {
2296			/* not the last component */
2297			link = walk_component(nd, WALK_MORE);
2298		}
2299		if (unlikely(link)) {
2300			if (IS_ERR(link))
2301				return PTR_ERR(link);
2302			/* a symlink to follow */
2303			nd->stack[depth++].name = name;
2304			name = link;
2305			continue;
2306		}
2307		if (unlikely(!d_can_lookup(nd->path.dentry))) {
2308			if (nd->flags & LOOKUP_RCU) {
2309				if (!try_to_unlazy(nd))
2310					return -ECHILD;
2311			}
2312			return -ENOTDIR;
2313		}
2314	}
2315}
2316
2317/* must be paired with terminate_walk() */
2318static const char *path_init(struct nameidata *nd, unsigned flags)
2319{
2320	int error;
2321	const char *s = nd->name->name;
2322
2323	/* LOOKUP_CACHED requires RCU, ask caller to retry */
2324	if ((flags & (LOOKUP_RCU | LOOKUP_CACHED)) == LOOKUP_CACHED)
2325		return ERR_PTR(-EAGAIN);
2326
2327	if (!*s)
2328		flags &= ~LOOKUP_RCU;
2329	if (flags & LOOKUP_RCU)
2330		rcu_read_lock();
2331
2332	nd->flags = flags;
2333	nd->state |= ND_JUMPED;
2334
2335	nd->m_seq = __read_seqcount_begin(&mount_lock.seqcount);
2336	nd->r_seq = __read_seqcount_begin(&rename_lock.seqcount);
2337	smp_rmb();
2338
2339	if (nd->state & ND_ROOT_PRESET) {
2340		struct dentry *root = nd->root.dentry;
2341		struct inode *inode = root->d_inode;
2342		if (*s && unlikely(!d_can_lookup(root)))
2343			return ERR_PTR(-ENOTDIR);
2344		nd->path = nd->root;
2345		nd->inode = inode;
2346		if (flags & LOOKUP_RCU) {
2347			nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
2348			nd->root_seq = nd->seq;
2349		} else {
2350			path_get(&nd->path);
2351		}
2352		return s;
2353	}
2354
2355	nd->root.mnt = NULL;
2356
2357	/* Absolute pathname -- fetch the root (LOOKUP_IN_ROOT uses nd->dfd). */
2358	if (*s == '/' && !(flags & LOOKUP_IN_ROOT)) {
2359		error = nd_jump_root(nd);
2360		if (unlikely(error))
2361			return ERR_PTR(error);
2362		return s;
2363	}
2364
2365	/* Relative pathname -- get the starting-point it is relative to. */
2366	if (nd->dfd == AT_FDCWD) {
2367		if (flags & LOOKUP_RCU) {
2368			struct fs_struct *fs = current->fs;
2369			unsigned seq;
2370
2371			do {
2372				seq = read_seqcount_begin(&fs->seq);
2373				nd->path = fs->pwd;
2374				nd->inode = nd->path.dentry->d_inode;
2375				nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
2376			} while (read_seqcount_retry(&fs->seq, seq));
2377		} else {
2378			get_fs_pwd(current->fs, &nd->path);
2379			nd->inode = nd->path.dentry->d_inode;
2380		}
2381	} else {
2382		/* Caller must check execute permissions on the starting path component */
2383		struct fd f = fdget_raw(nd->dfd);
2384		struct dentry *dentry;
2385
2386		if (!f.file)
2387			return ERR_PTR(-EBADF);
2388
2389		dentry = f.file->f_path.dentry;
2390
2391		if (*s && unlikely(!d_can_lookup(dentry))) {
2392			fdput(f);
2393			return ERR_PTR(-ENOTDIR);
2394		}
2395
2396		nd->path = f.file->f_path;
2397		if (flags & LOOKUP_RCU) {
2398			nd->inode = nd->path.dentry->d_inode;
2399			nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
2400		} else {
2401			path_get(&nd->path);
2402			nd->inode = nd->path.dentry->d_inode;
2403		}
2404		fdput(f);
2405	}
2406
2407	/* For scoped-lookups we need to set the root to the dirfd as well. */
2408	if (flags & LOOKUP_IS_SCOPED) {
2409		nd->root = nd->path;
2410		if (flags & LOOKUP_RCU) {
2411			nd->root_seq = nd->seq;
2412		} else {
2413			path_get(&nd->root);
2414			nd->state |= ND_ROOT_GRABBED;
2415		}
2416	}
2417	return s;
2418}
2419
2420static inline const char *lookup_last(struct nameidata *nd)
2421{
2422	if (nd->last_type == LAST_NORM && nd->last.name[nd->last.len])
2423		nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
2424
2425	return walk_component(nd, WALK_TRAILING);
2426}
2427
2428static int handle_lookup_down(struct nameidata *nd)
2429{
2430	if (!(nd->flags & LOOKUP_RCU))
2431		dget(nd->path.dentry);
2432	return PTR_ERR(step_into(nd, WALK_NOFOLLOW,
2433			nd->path.dentry, nd->inode, nd->seq));
2434}
2435
2436/* Returns 0 and nd will be valid on success; Retuns error, otherwise. */
2437static int path_lookupat(struct nameidata *nd, unsigned flags, struct path *path)
2438{
2439	const char *s = path_init(nd, flags);
2440	int err;
2441
2442	if (unlikely(flags & LOOKUP_DOWN) && !IS_ERR(s)) {
2443		err = handle_lookup_down(nd);
2444		if (unlikely(err < 0))
2445			s = ERR_PTR(err);
2446	}
2447
2448	while (!(err = link_path_walk(s, nd)) &&
2449	       (s = lookup_last(nd)) != NULL)
2450		;
2451	if (!err && unlikely(nd->flags & LOOKUP_MOUNTPOINT)) {
2452		err = handle_lookup_down(nd);
2453		nd->state &= ~ND_JUMPED; // no d_weak_revalidate(), please...
2454	}
2455	if (!err)
2456		err = complete_walk(nd);
2457
2458	if (!err && nd->flags & LOOKUP_DIRECTORY)
2459		if (!d_can_lookup(nd->path.dentry))
2460			err = -ENOTDIR;
2461	if (!err) {
2462		*path = nd->path;
2463		nd->path.mnt = NULL;
2464		nd->path.dentry = NULL;
2465	}
2466	terminate_walk(nd);
2467	return err;
2468}
2469
2470int filename_lookup(int dfd, struct filename *name, unsigned flags,
2471		    struct path *path, struct path *root)
2472{
2473	int retval;
2474	struct nameidata nd;
2475	if (IS_ERR(name))
2476		return PTR_ERR(name);
2477	set_nameidata(&nd, dfd, name, root);
2478	retval = path_lookupat(&nd, flags | LOOKUP_RCU, path);
2479	if (unlikely(retval == -ECHILD))
2480		retval = path_lookupat(&nd, flags, path);
2481	if (unlikely(retval == -ESTALE))
2482		retval = path_lookupat(&nd, flags | LOOKUP_REVAL, path);
2483
2484	if (likely(!retval))
2485		audit_inode(name, path->dentry,
2486			    flags & LOOKUP_MOUNTPOINT ? AUDIT_INODE_NOEVAL : 0);
2487	restore_nameidata();
2488	return retval;
2489}
2490
2491/* Returns 0 and nd will be valid on success; Retuns error, otherwise. */
2492static int path_parentat(struct nameidata *nd, unsigned flags,
2493				struct path *parent)
2494{
2495	const char *s = path_init(nd, flags);
2496	int err = link_path_walk(s, nd);
2497	if (!err)
2498		err = complete_walk(nd);
2499	if (!err) {
2500		*parent = nd->path;
2501		nd->path.mnt = NULL;
2502		nd->path.dentry = NULL;
2503	}
2504	terminate_walk(nd);
2505	return err;
2506}
2507
2508/* Note: this does not consume "name" */
2509static int filename_parentat(int dfd, struct filename *name,
2510			     unsigned int flags, struct path *parent,
2511			     struct qstr *last, int *type)
2512{
2513	int retval;
2514	struct nameidata nd;
2515
2516	if (IS_ERR(name))
2517		return PTR_ERR(name);
2518	set_nameidata(&nd, dfd, name, NULL);
2519	retval = path_parentat(&nd, flags | LOOKUP_RCU, parent);
2520	if (unlikely(retval == -ECHILD))
2521		retval = path_parentat(&nd, flags, parent);
2522	if (unlikely(retval == -ESTALE))
2523		retval = path_parentat(&nd, flags | LOOKUP_REVAL, parent);
2524	if (likely(!retval)) {
2525		*last = nd.last;
2526		*type = nd.last_type;
2527		audit_inode(name, parent->dentry, AUDIT_INODE_PARENT);
2528	}
2529	restore_nameidata();
2530	return retval;
2531}
2532
2533/* does lookup, returns the object with parent locked */
2534static struct dentry *__kern_path_locked(struct filename *name, struct path *path)
2535{
2536	struct dentry *d;
2537	struct qstr last;
2538	int type, error;
2539
2540	error = filename_parentat(AT_FDCWD, name, 0, path, &last, &type);
2541	if (error)
2542		return ERR_PTR(error);
2543	if (unlikely(type != LAST_NORM)) {
2544		path_put(path);
2545		return ERR_PTR(-EINVAL);
2546	}
2547	inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
2548	d = __lookup_hash(&last, path->dentry, 0);
2549	if (IS_ERR(d)) {
2550		inode_unlock(path->dentry->d_inode);
2551		path_put(path);
2552	}
2553	return d;
2554}
2555
2556struct dentry *kern_path_locked(const char *name, struct path *path)
2557{
2558	struct filename *filename = getname_kernel(name);
2559	struct dentry *res = __kern_path_locked(filename, path);
2560
2561	putname(filename);
2562	return res;
2563}
2564
2565int kern_path(const char *name, unsigned int flags, struct path *path)
2566{
2567	struct filename *filename = getname_kernel(name);
2568	int ret = filename_lookup(AT_FDCWD, filename, flags, path, NULL);
2569
2570	putname(filename);
2571	return ret;
2572
2573}
2574EXPORT_SYMBOL(kern_path);
2575
2576/**
2577 * vfs_path_lookup - lookup a file path relative to a dentry-vfsmount pair
2578 * @dentry:  pointer to dentry of the base directory
2579 * @mnt: pointer to vfs mount of the base directory
2580 * @name: pointer to file name
2581 * @flags: lookup flags
2582 * @path: pointer to struct path to fill
2583 */
2584int vfs_path_lookup(struct dentry *dentry, struct vfsmount *mnt,
2585		    const char *name, unsigned int flags,
2586		    struct path *path)
2587{
2588	struct filename *filename;
2589	struct path root = {.mnt = mnt, .dentry = dentry};
2590	int ret;
2591
2592	filename = getname_kernel(name);
2593	/* the first argument of filename_lookup() is ignored with root */
2594	ret = filename_lookup(AT_FDCWD, filename, flags, path, &root);
2595	putname(filename);
2596	return ret;
2597}
2598EXPORT_SYMBOL(vfs_path_lookup);
2599
2600static int lookup_one_common(struct user_namespace *mnt_userns,
2601			     const char *name, struct dentry *base, int len,
2602			     struct qstr *this)
2603{
2604	this->name = name;
2605	this->len = len;
2606	this->hash = full_name_hash(base, name, len);
2607	if (!len)
2608		return -EACCES;
2609
2610	if (unlikely(name[0] == '.')) {
2611		if (len < 2 || (len == 2 && name[1] == '.'))
2612			return -EACCES;
2613	}
2614
2615	while (len--) {
2616		unsigned int c = *(const unsigned char *)name++;
2617		if (c == '/' || c == '\0')
2618			return -EACCES;
2619	}
2620	/*
2621	 * See if the low-level filesystem might want
2622	 * to use its own hash..
2623	 */
2624	if (base->d_flags & DCACHE_OP_HASH) {
2625		int err = base->d_op->d_hash(base, this);
2626		if (err < 0)
2627			return err;
2628	}
2629
2630	return inode_permission(mnt_userns, base->d_inode, MAY_EXEC);
2631}
2632
2633/**
2634 * try_lookup_one_len - filesystem helper to lookup single pathname component
2635 * @name:	pathname component to lookup
2636 * @base:	base directory to lookup from
2637 * @len:	maximum length @len should be interpreted to
2638 *
2639 * Look up a dentry by name in the dcache, returning NULL if it does not
2640 * currently exist.  The function does not try to create a dentry.
2641 *
2642 * Note that this routine is purely a helper for filesystem usage and should
2643 * not be called by generic code.
2644 *
2645 * The caller must hold base->i_mutex.
2646 */
2647struct dentry *try_lookup_one_len(const char *name, struct dentry *base, int len)
2648{
2649	struct qstr this;
2650	int err;
2651
2652	WARN_ON_ONCE(!inode_is_locked(base->d_inode));
2653
2654	err = lookup_one_common(&init_user_ns, name, base, len, &this);
2655	if (err)
2656		return ERR_PTR(err);
2657
2658	return lookup_dcache(&this, base, 0);
2659}
2660EXPORT_SYMBOL(try_lookup_one_len);
2661
2662/**
2663 * lookup_one_len - filesystem helper to lookup single pathname component
2664 * @name:	pathname component to lookup
2665 * @base:	base directory to lookup from
2666 * @len:	maximum length @len should be interpreted to
2667 *
2668 * Note that this routine is purely a helper for filesystem usage and should
2669 * not be called by generic code.
2670 *
2671 * The caller must hold base->i_mutex.
2672 */
2673struct dentry *lookup_one_len(const char *name, struct dentry *base, int len)
2674{
2675	struct dentry *dentry;
2676	struct qstr this;
2677	int err;
2678
2679	WARN_ON_ONCE(!inode_is_locked(base->d_inode));
2680
2681	err = lookup_one_common(&init_user_ns, name, base, len, &this);
2682	if (err)
2683		return ERR_PTR(err);
2684
2685	dentry = lookup_dcache(&this, base, 0);
2686	return dentry ? dentry : __lookup_slow(&this, base, 0);
2687}
2688EXPORT_SYMBOL(lookup_one_len);
2689
2690/**
2691 * lookup_one - filesystem helper to lookup single pathname component
2692 * @mnt_userns:	user namespace of the mount the lookup is performed from
2693 * @name:	pathname component to lookup
2694 * @base:	base directory to lookup from
2695 * @len:	maximum length @len should be interpreted to
2696 *
2697 * Note that this routine is purely a helper for filesystem usage and should
2698 * not be called by generic code.
2699 *
2700 * The caller must hold base->i_mutex.
2701 */
2702struct dentry *lookup_one(struct user_namespace *mnt_userns, const char *name,
2703			  struct dentry *base, int len)
2704{
2705	struct dentry *dentry;
2706	struct qstr this;
2707	int err;
2708
2709	WARN_ON_ONCE(!inode_is_locked(base->d_inode));
2710
2711	err = lookup_one_common(mnt_userns, name, base, len, &this);
2712	if (err)
2713		return ERR_PTR(err);
2714
2715	dentry = lookup_dcache(&this, base, 0);
2716	return dentry ? dentry : __lookup_slow(&this, base, 0);
2717}
2718EXPORT_SYMBOL(lookup_one);
2719
2720/**
2721 * lookup_one_len_unlocked - filesystem helper to lookup single pathname component
2722 * @name:	pathname component to lookup
2723 * @base:	base directory to lookup from
2724 * @len:	maximum length @len should be interpreted to
2725 *
2726 * Note that this routine is purely a helper for filesystem usage and should
2727 * not be called by generic code.
2728 *
2729 * Unlike lookup_one_len, it should be called without the parent
2730 * i_mutex held, and will take the i_mutex itself if necessary.
2731 */
2732struct dentry *lookup_one_len_unlocked(const char *name,
2733				       struct dentry *base, int len)
2734{
2735	struct qstr this;
2736	int err;
2737	struct dentry *ret;
2738
2739	err = lookup_one_common(&init_user_ns, name, base, len, &this);
2740	if (err)
2741		return ERR_PTR(err);
2742
2743	ret = lookup_dcache(&this, base, 0);
2744	if (!ret)
2745		ret = lookup_slow(&this, base, 0);
2746	return ret;
2747}
2748EXPORT_SYMBOL(lookup_one_len_unlocked);
2749
2750/*
2751 * Like lookup_one_len_unlocked(), except that it yields ERR_PTR(-ENOENT)
2752 * on negatives.  Returns known positive or ERR_PTR(); that's what
2753 * most of the users want.  Note that pinned negative with unlocked parent
2754 * _can_ become positive at any time, so callers of lookup_one_len_unlocked()
2755 * need to be very careful; pinned positives have ->d_inode stable, so
2756 * this one avoids such problems.
2757 */
2758struct dentry *lookup_positive_unlocked(const char *name,
2759				       struct dentry *base, int len)
2760{
2761	struct dentry *ret = lookup_one_len_unlocked(name, base, len);
2762	if (!IS_ERR(ret) && d_flags_negative(smp_load_acquire(&ret->d_flags))) {
2763		dput(ret);
2764		ret = ERR_PTR(-ENOENT);
2765	}
2766	return ret;
2767}
2768EXPORT_SYMBOL(lookup_positive_unlocked);
2769
2770#ifdef CONFIG_UNIX98_PTYS
2771int path_pts(struct path *path)
2772{
2773	/* Find something mounted on "pts" in the same directory as
2774	 * the input path.
2775	 */
2776	struct dentry *parent = dget_parent(path->dentry);
2777	struct dentry *child;
2778	struct qstr this = QSTR_INIT("pts", 3);
2779
2780	if (unlikely(!path_connected(path->mnt, parent))) {
2781		dput(parent);
2782		return -ENOENT;
2783	}
2784	dput(path->dentry);
2785	path->dentry = parent;
2786	child = d_hash_and_lookup(parent, &this);
2787	if (!child)
2788		return -ENOENT;
2789
2790	path->dentry = child;
2791	dput(parent);
2792	follow_down(path);
2793	return 0;
2794}
2795#endif
2796
2797int user_path_at_empty(int dfd, const char __user *name, unsigned flags,
2798		 struct path *path, int *empty)
2799{
2800	struct filename *filename = getname_flags(name, flags, empty);
2801	int ret = filename_lookup(dfd, filename, flags, path, NULL);
2802
2803	putname(filename);
2804	return ret;
2805}
2806EXPORT_SYMBOL(user_path_at_empty);
2807
2808int __check_sticky(struct user_namespace *mnt_userns, struct inode *dir,
2809		   struct inode *inode)
2810{
2811	kuid_t fsuid = current_fsuid();
2812
2813	if (uid_eq(i_uid_into_mnt(mnt_userns, inode), fsuid))
2814		return 0;
2815	if (uid_eq(i_uid_into_mnt(mnt_userns, dir), fsuid))
2816		return 0;
2817	return !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FOWNER);
2818}
2819EXPORT_SYMBOL(__check_sticky);
2820
2821/*
2822 *	Check whether we can remove a link victim from directory dir, check
2823 *  whether the type of victim is right.
2824 *  1. We can't do it if dir is read-only (done in permission())
2825 *  2. We should have write and exec permissions on dir
2826 *  3. We can't remove anything from append-only dir
2827 *  4. We can't do anything with immutable dir (done in permission())
2828 *  5. If the sticky bit on dir is set we should either
2829 *	a. be owner of dir, or
2830 *	b. be owner of victim, or
2831 *	c. have CAP_FOWNER capability
2832 *  6. If the victim is append-only or immutable we can't do antyhing with
2833 *     links pointing to it.
2834 *  7. If the victim has an unknown uid or gid we can't change the inode.
2835 *  8. If we were asked to remove a directory and victim isn't one - ENOTDIR.
2836 *  9. If we were asked to remove a non-directory and victim isn't one - EISDIR.
2837 * 10. We can't remove a root or mountpoint.
2838 * 11. We don't allow removal of NFS sillyrenamed files; it's handled by
2839 *     nfs_async_unlink().
2840 */
2841static int may_delete(struct user_namespace *mnt_userns, struct inode *dir,
2842		      struct dentry *victim, bool isdir)
2843{
2844	struct inode *inode = d_backing_inode(victim);
2845	int error;
2846
2847	if (d_is_negative(victim))
2848		return -ENOENT;
2849	BUG_ON(!inode);
2850
2851	BUG_ON(victim->d_parent->d_inode != dir);
2852
2853	/* Inode writeback is not safe when the uid or gid are invalid. */
2854	if (!uid_valid(i_uid_into_mnt(mnt_userns, inode)) ||
2855	    !gid_valid(i_gid_into_mnt(mnt_userns, inode)))
2856		return -EOVERFLOW;
2857
2858	audit_inode_child(dir, victim, AUDIT_TYPE_CHILD_DELETE);
2859
2860	error = inode_permission(mnt_userns, dir, MAY_WRITE | MAY_EXEC);
2861	if (error)
2862		return error;
2863	if (IS_APPEND(dir))
2864		return -EPERM;
2865
2866	if (check_sticky(mnt_userns, dir, inode) || IS_APPEND(inode) ||
2867	    IS_IMMUTABLE(inode) || IS_SWAPFILE(inode) ||
2868	    HAS_UNMAPPED_ID(mnt_userns, inode))
2869		return -EPERM;
2870	if (isdir) {
2871		if (!d_is_dir(victim))
2872			return -ENOTDIR;
2873		if (IS_ROOT(victim))
2874			return -EBUSY;
2875	} else if (d_is_dir(victim))
2876		return -EISDIR;
2877	if (IS_DEADDIR(dir))
2878		return -ENOENT;
2879	if (victim->d_flags & DCACHE_NFSFS_RENAMED)
2880		return -EBUSY;
2881	return 0;
2882}
2883
2884/*	Check whether we can create an object with dentry child in directory
2885 *  dir.
2886 *  1. We can't do it if child already exists (open has special treatment for
2887 *     this case, but since we are inlined it's OK)
2888 *  2. We can't do it if dir is read-only (done in permission())
2889 *  3. We can't do it if the fs can't represent the fsuid or fsgid.
2890 *  4. We should have write and exec permissions on dir
2891 *  5. We can't do it if dir is immutable (done in permission())
2892 */
2893static inline int may_create(struct user_namespace *mnt_userns,
2894			     struct inode *dir, struct dentry *child)
2895{
2896	audit_inode_child(dir, child, AUDIT_TYPE_CHILD_CREATE);
2897	if (child->d_inode)
2898		return -EEXIST;
2899	if (IS_DEADDIR(dir))
2900		return -ENOENT;
2901	if (!fsuidgid_has_mapping(dir->i_sb, mnt_userns))
2902		return -EOVERFLOW;
2903
2904	return inode_permission(mnt_userns, dir, MAY_WRITE | MAY_EXEC);
2905}
2906
2907/*
2908 * p1 and p2 should be directories on the same fs.
2909 */
2910struct dentry *lock_rename(struct dentry *p1, struct dentry *p2)
2911{
2912	struct dentry *p;
2913
2914	if (p1 == p2) {
2915		inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
2916		return NULL;
2917	}
2918
2919	mutex_lock(&p1->d_sb->s_vfs_rename_mutex);
2920
2921	p = d_ancestor(p2, p1);
2922	if (p) {
2923		inode_lock_nested(p2->d_inode, I_MUTEX_PARENT);
2924		inode_lock_nested(p1->d_inode, I_MUTEX_CHILD);
2925		return p;
2926	}
2927
2928	p = d_ancestor(p1, p2);
2929	if (p) {
2930		inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
2931		inode_lock_nested(p2->d_inode, I_MUTEX_CHILD);
2932		return p;
2933	}
2934
2935	inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
2936	inode_lock_nested(p2->d_inode, I_MUTEX_PARENT2);
2937	return NULL;
2938}
2939EXPORT_SYMBOL(lock_rename);
2940
2941void unlock_rename(struct dentry *p1, struct dentry *p2)
2942{
2943	inode_unlock(p1->d_inode);
2944	if (p1 != p2) {
2945		inode_unlock(p2->d_inode);
2946		mutex_unlock(&p1->d_sb->s_vfs_rename_mutex);
2947	}
2948}
2949EXPORT_SYMBOL(unlock_rename);
2950
2951/**
2952 * vfs_create - create new file
2953 * @mnt_userns:	user namespace of the mount the inode was found from
2954 * @dir:	inode of @dentry
2955 * @dentry:	pointer to dentry of the base directory
2956 * @mode:	mode of the new file
2957 * @want_excl:	whether the file must not yet exist
2958 *
2959 * Create a new file.
2960 *
2961 * If the inode has been found through an idmapped mount the user namespace of
2962 * the vfsmount must be passed through @mnt_userns. This function will then take
2963 * care to map the inode according to @mnt_userns before checking permissions.
2964 * On non-idmapped mounts or if permission checking is to be performed on the
2965 * raw inode simply passs init_user_ns.
2966 */
2967int vfs_create(struct user_namespace *mnt_userns, struct inode *dir,
2968	       struct dentry *dentry, umode_t mode, bool want_excl)
2969{
2970	int error = may_create(mnt_userns, dir, dentry);
2971	if (error)
2972		return error;
2973
2974	if (!dir->i_op->create)
2975		return -EACCES;	/* shouldn't it be ENOSYS? */
2976	mode &= S_IALLUGO;
2977	mode |= S_IFREG;
2978	error = security_inode_create(dir, dentry, mode);
2979	if (error)
2980		return error;
2981	error = dir->i_op->create(mnt_userns, dir, dentry, mode, want_excl);
2982	if (!error)
2983		fsnotify_create(dir, dentry);
2984	return error;
2985}
2986EXPORT_SYMBOL(vfs_create);
2987
2988int vfs_mkobj(struct dentry *dentry, umode_t mode,
2989		int (*f)(struct dentry *, umode_t, void *),
2990		void *arg)
2991{
2992	struct inode *dir = dentry->d_parent->d_inode;
2993	int error = may_create(&init_user_ns, dir, dentry);
2994	if (error)
2995		return error;
2996
2997	mode &= S_IALLUGO;
2998	mode |= S_IFREG;
2999	error = security_inode_create(dir, dentry, mode);
3000	if (error)
3001		return error;
3002	error = f(dentry, mode, arg);
3003	if (!error)
3004		fsnotify_create(dir, dentry);
3005	return error;
3006}
3007EXPORT_SYMBOL(vfs_mkobj);
3008
3009bool may_open_dev(const struct path *path)
3010{
3011	return !(path->mnt->mnt_flags & MNT_NODEV) &&
3012		!(path->mnt->mnt_sb->s_iflags & SB_I_NODEV);
3013}
3014
3015static int may_open(struct user_namespace *mnt_userns, const struct path *path,
3016		    int acc_mode, int flag)
3017{
3018	struct dentry *dentry = path->dentry;
3019	struct inode *inode = dentry->d_inode;
3020	int error;
3021
3022	if (!inode)
3023		return -ENOENT;
3024
3025	switch (inode->i_mode & S_IFMT) {
3026	case S_IFLNK:
3027		return -ELOOP;
3028	case S_IFDIR:
3029		if (acc_mode & MAY_WRITE)
3030			return -EISDIR;
3031		if (acc_mode & MAY_EXEC)
3032			return -EACCES;
3033		break;
3034	case S_IFBLK:
3035	case S_IFCHR:
3036		if (!may_open_dev(path))
3037			return -EACCES;
3038		fallthrough;
3039	case S_IFIFO:
3040	case S_IFSOCK:
3041		if (acc_mode & MAY_EXEC)
3042			return -EACCES;
3043		flag &= ~O_TRUNC;
3044		break;
3045	case S_IFREG:
3046		if ((acc_mode & MAY_EXEC) && path_noexec(path))
3047			return -EACCES;
3048		break;
3049	}
3050
3051	error = inode_permission(mnt_userns, inode, MAY_OPEN | acc_mode);
3052	if (error)
3053		return error;
3054
3055	/*
3056	 * An append-only file must be opened in append mode for writing.
3057	 */
3058	if (IS_APPEND(inode)) {
3059		if  ((flag & O_ACCMODE) != O_RDONLY && !(flag & O_APPEND))
3060			return -EPERM;
3061		if (flag & O_TRUNC)
3062			return -EPERM;
3063	}
3064
3065	/* O_NOATIME can only be set by the owner or superuser */
3066	if (flag & O_NOATIME && !inode_owner_or_capable(mnt_userns, inode))
3067		return -EPERM;
3068
3069	return 0;
3070}
3071
3072static int handle_truncate(struct user_namespace *mnt_userns, struct file *filp)
3073{
3074	const struct path *path = &filp->f_path;
3075	struct inode *inode = path->dentry->d_inode;
3076	int error = get_write_access(inode);
3077	if (error)
3078		return error;
3079	/*
3080	 * Refuse to truncate files with mandatory locks held on them.
3081	 */
3082	error = security_path_truncate(path);
3083	if (!error) {
3084		error = do_truncate(mnt_userns, path->dentry, 0,
3085				    ATTR_MTIME|ATTR_CTIME|ATTR_OPEN,
3086				    filp);
3087	}
3088	put_write_access(inode);
3089	return error;
3090}
3091
3092static inline int open_to_namei_flags(int flag)
3093{
3094	if ((flag & O_ACCMODE) == 3)
3095		flag--;
3096	return flag;
3097}
3098
3099static int may_o_create(struct user_namespace *mnt_userns,
3100			const struct path *dir, struct dentry *dentry,
3101			umode_t mode)
3102{
3103	int error = security_path_mknod(dir, dentry, mode, 0);
3104	if (error)
3105		return error;
3106
3107	if (!fsuidgid_has_mapping(dir->dentry->d_sb, mnt_userns))
3108		return -EOVERFLOW;
3109
3110	error = inode_permission(mnt_userns, dir->dentry->d_inode,
3111				 MAY_WRITE | MAY_EXEC);
3112	if (error)
3113		return error;
3114
3115	return security_inode_create(dir->dentry->d_inode, dentry, mode);
3116}
3117
3118/*
3119 * Attempt to atomically look up, create and open a file from a negative
3120 * dentry.
3121 *
3122 * Returns 0 if successful.  The file will have been created and attached to
3123 * @file by the filesystem calling finish_open().
3124 *
3125 * If the file was looked up only or didn't need creating, FMODE_OPENED won't
3126 * be set.  The caller will need to perform the open themselves.  @path will
3127 * have been updated to point to the new dentry.  This may be negative.
3128 *
3129 * Returns an error code otherwise.
3130 */
3131static struct dentry *atomic_open(struct nameidata *nd, struct dentry *dentry,
3132				  struct file *file,
3133				  int open_flag, umode_t mode)
3134{
3135	struct dentry *const DENTRY_NOT_SET = (void *) -1UL;
3136	struct inode *dir =  nd->path.dentry->d_inode;
3137	int error;
3138
3139	if (nd->flags & LOOKUP_DIRECTORY)
3140		open_flag |= O_DIRECTORY;
3141
3142	file->f_path.dentry = DENTRY_NOT_SET;
3143	file->f_path.mnt = nd->path.mnt;
3144	error = dir->i_op->atomic_open(dir, dentry, file,
3145				       open_to_namei_flags(open_flag), mode);
3146	d_lookup_done(dentry);
3147	if (!error) {
3148		if (file->f_mode & FMODE_OPENED) {
3149			if (unlikely(dentry != file->f_path.dentry)) {
3150				dput(dentry);
3151				dentry = dget(file->f_path.dentry);
3152			}
3153		} else if (WARN_ON(file->f_path.dentry == DENTRY_NOT_SET)) {
3154			error = -EIO;
3155		} else {
3156			if (file->f_path.dentry) {
3157				dput(dentry);
3158				dentry = file->f_path.dentry;
3159			}
3160			if (unlikely(d_is_negative(dentry)))
3161				error = -ENOENT;
3162		}
3163	}
3164	if (error) {
3165		dput(dentry);
3166		dentry = ERR_PTR(error);
3167	}
3168	return dentry;
3169}
3170
3171/*
3172 * Look up and maybe create and open the last component.
3173 *
3174 * Must be called with parent locked (exclusive in O_CREAT case).
3175 *
3176 * Returns 0 on success, that is, if
3177 *  the file was successfully atomically created (if necessary) and opened, or
3178 *  the file was not completely opened at this time, though lookups and
3179 *  creations were performed.
3180 * These case are distinguished by presence of FMODE_OPENED on file->f_mode.
3181 * In the latter case dentry returned in @path might be negative if O_CREAT
3182 * hadn't been specified.
3183 *
3184 * An error code is returned on failure.
3185 */
3186static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
3187				  const struct open_flags *op,
3188				  bool got_write)
3189{
3190	struct user_namespace *mnt_userns;
3191	struct dentry *dir = nd->path.dentry;
3192	struct inode *dir_inode = dir->d_inode;
3193	int open_flag = op->open_flag;
3194	struct dentry *dentry;
3195	int error, create_error = 0;
3196	umode_t mode = op->mode;
3197	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
3198
3199	if (unlikely(IS_DEADDIR(dir_inode)))
3200		return ERR_PTR(-ENOENT);
3201
3202	file->f_mode &= ~FMODE_CREATED;
3203	dentry = d_lookup(dir, &nd->last);
3204	for (;;) {
3205		if (!dentry) {
3206			dentry = d_alloc_parallel(dir, &nd->last, &wq);
3207			if (IS_ERR(dentry))
3208				return dentry;
3209		}
3210		if (d_in_lookup(dentry))
3211			break;
3212
3213		error = d_revalidate(dentry, nd->flags);
3214		if (likely(error > 0))
3215			break;
3216		if (error)
3217			goto out_dput;
3218		d_invalidate(dentry);
3219		dput(dentry);
3220		dentry = NULL;
3221	}
3222	if (dentry->d_inode) {
3223		/* Cached positive dentry: will open in f_op->open */
3224		return dentry;
3225	}
3226
3227	/*
3228	 * Checking write permission is tricky, bacuse we don't know if we are
3229	 * going to actually need it: O_CREAT opens should work as long as the
3230	 * file exists.  But checking existence breaks atomicity.  The trick is
3231	 * to check access and if not granted clear O_CREAT from the flags.
3232	 *
3233	 * Another problem is returing the "right" error value (e.g. for an
3234	 * O_EXCL open we want to return EEXIST not EROFS).
3235	 */
3236	if (unlikely(!got_write))
3237		open_flag &= ~O_TRUNC;
3238	mnt_userns = mnt_user_ns(nd->path.mnt);
3239	if (open_flag & O_CREAT) {
3240		if (open_flag & O_EXCL)
3241			open_flag &= ~O_TRUNC;
3242		if (!IS_POSIXACL(dir->d_inode))
3243			mode &= ~current_umask();
3244		if (likely(got_write))
3245			create_error = may_o_create(mnt_userns, &nd->path,
3246						    dentry, mode);
3247		else
3248			create_error = -EROFS;
3249	}
3250	if (create_error)
3251		open_flag &= ~O_CREAT;
3252	if (dir_inode->i_op->atomic_open) {
3253		dentry = atomic_open(nd, dentry, file, open_flag, mode);
3254		if (unlikely(create_error) && dentry == ERR_PTR(-ENOENT))
3255			dentry = ERR_PTR(create_error);
3256		return dentry;
3257	}
3258
3259	if (d_in_lookup(dentry)) {
3260		struct dentry *res = dir_inode->i_op->lookup(dir_inode, dentry,
3261							     nd->flags);
3262		d_lookup_done(dentry);
3263		if (unlikely(res)) {
3264			if (IS_ERR(res)) {
3265				error = PTR_ERR(res);
3266				goto out_dput;
3267			}
3268			dput(dentry);
3269			dentry = res;
3270		}
3271	}
3272
3273	/* Negative dentry, just create the file */
3274	if (!dentry->d_inode && (open_flag & O_CREAT)) {
3275		file->f_mode |= FMODE_CREATED;
3276		audit_inode_child(dir_inode, dentry, AUDIT_TYPE_CHILD_CREATE);
3277		if (!dir_inode->i_op->create) {
3278			error = -EACCES;
3279			goto out_dput;
3280		}
3281
3282		error = dir_inode->i_op->create(mnt_userns, dir_inode, dentry,
3283						mode, open_flag & O_EXCL);
3284		if (error)
3285			goto out_dput;
3286	}
3287	if (unlikely(create_error) && !dentry->d_inode) {
3288		error = create_error;
3289		goto out_dput;
3290	}
3291	return dentry;
3292
3293out_dput:
3294	dput(dentry);
3295	return ERR_PTR(error);
3296}
3297
3298static const char *open_last_lookups(struct nameidata *nd,
3299		   struct file *file, const struct open_flags *op)
3300{
3301	struct dentry *dir = nd->path.dentry;
3302	int open_flag = op->open_flag;
3303	bool got_write = false;
3304	unsigned seq;
3305	struct inode *inode;
3306	struct dentry *dentry;
3307	const char *res;
3308
3309	nd->flags |= op->intent;
3310
3311	if (nd->last_type != LAST_NORM) {
3312		if (nd->depth)
3313			put_link(nd);
3314		return handle_dots(nd, nd->last_type);
3315	}
3316
3317	if (!(open_flag & O_CREAT)) {
3318		if (nd->last.name[nd->last.len])
3319			nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
3320		/* we _can_ be in RCU mode here */
3321		dentry = lookup_fast(nd, &inode, &seq);
3322		if (IS_ERR(dentry))
3323			return ERR_CAST(dentry);
3324		if (likely(dentry))
3325			goto finish_lookup;
3326
3327		BUG_ON(nd->flags & LOOKUP_RCU);
3328	} else {
3329		/* create side of things */
3330		if (nd->flags & LOOKUP_RCU) {
3331			if (!try_to_unlazy(nd))
3332				return ERR_PTR(-ECHILD);
3333		}
3334		audit_inode(nd->name, dir, AUDIT_INODE_PARENT);
3335		/* trailing slashes? */
3336		if (unlikely(nd->last.name[nd->last.len]))
3337			return ERR_PTR(-EISDIR);
3338	}
3339
3340	if (open_flag & (O_CREAT | O_TRUNC | O_WRONLY | O_RDWR)) {
3341		got_write = !mnt_want_write(nd->path.mnt);
3342		/*
3343		 * do _not_ fail yet - we might not need that or fail with
3344		 * a different error; let lookup_open() decide; we'll be
3345		 * dropping this one anyway.
3346		 */
3347	}
3348	if (open_flag & O_CREAT)
3349		inode_lock(dir->d_inode);
3350	else
3351		inode_lock_shared(dir->d_inode);
3352	dentry = lookup_open(nd, file, op, got_write);
3353	if (!IS_ERR(dentry) && (file->f_mode & FMODE_CREATED))
3354		fsnotify_create(dir->d_inode, dentry);
3355	if (open_flag & O_CREAT)
3356		inode_unlock(dir->d_inode);
3357	else
3358		inode_unlock_shared(dir->d_inode);
3359
3360	if (got_write)
3361		mnt_drop_write(nd->path.mnt);
3362
3363	if (IS_ERR(dentry))
3364		return ERR_CAST(dentry);
3365
3366	if (file->f_mode & (FMODE_OPENED | FMODE_CREATED)) {
3367		dput(nd->path.dentry);
3368		nd->path.dentry = dentry;
3369		return NULL;
3370	}
3371
3372finish_lookup:
3373	if (nd->depth)
3374		put_link(nd);
3375	res = step_into(nd, WALK_TRAILING, dentry, inode, seq);
3376	if (unlikely(res))
3377		nd->flags &= ~(LOOKUP_OPEN|LOOKUP_CREATE|LOOKUP_EXCL);
3378	return res;
3379}
3380
3381/*
3382 * Handle the last step of open()
3383 */
3384static int do_open(struct nameidata *nd,
3385		   struct file *file, const struct open_flags *op)
3386{
3387	struct user_namespace *mnt_userns;
3388	int open_flag = op->open_flag;
3389	bool do_truncate;
3390	int acc_mode;
3391	int error;
3392
3393	if (!(file->f_mode & (FMODE_OPENED | FMODE_CREATED))) {
3394		error = complete_walk(nd);
3395		if (error)
3396			return error;
3397	}
3398	if (!(file->f_mode & FMODE_CREATED))
3399		audit_inode(nd->name, nd->path.dentry, 0);
3400	mnt_userns = mnt_user_ns(nd->path.mnt);
3401	if (open_flag & O_CREAT) {
3402		if ((open_flag & O_EXCL) && !(file->f_mode & FMODE_CREATED))
3403			return -EEXIST;
3404		if (d_is_dir(nd->path.dentry))
3405			return -EISDIR;
3406		error = may_create_in_sticky(mnt_userns, nd,
3407					     d_backing_inode(nd->path.dentry));
3408		if (unlikely(error))
3409			return error;
3410	}
3411	if ((nd->flags & LOOKUP_DIRECTORY) && !d_can_lookup(nd->path.dentry))
3412		return -ENOTDIR;
3413
3414	do_truncate = false;
3415	acc_mode = op->acc_mode;
3416	if (file->f_mode & FMODE_CREATED) {
3417		/* Don't check for write permission, don't truncate */
3418		open_flag &= ~O_TRUNC;
3419		acc_mode = 0;
3420	} else if (d_is_reg(nd->path.dentry) && open_flag & O_TRUNC) {
3421		error = mnt_want_write(nd->path.mnt);
3422		if (error)
3423			return error;
3424		do_truncate = true;
3425	}
3426	error = may_open(mnt_userns, &nd->path, acc_mode, open_flag);
3427	if (!error && !(file->f_mode & FMODE_OPENED))
3428		error = vfs_open(&nd->path, file);
3429	if (!error)
3430		error = ima_file_check(file, op->acc_mode);
3431	if (!error && do_truncate)
3432		error = handle_truncate(mnt_userns, file);
3433	if (unlikely(error > 0)) {
3434		WARN_ON(1);
3435		error = -EINVAL;
3436	}
3437	if (do_truncate)
3438		mnt_drop_write(nd->path.mnt);
3439	return error;
3440}
3441
3442/**
3443 * vfs_tmpfile - create tmpfile
3444 * @mnt_userns:	user namespace of the mount the inode was found from
3445 * @dentry:	pointer to dentry of the base directory
3446 * @mode:	mode of the new tmpfile
3447 * @open_flag:	flags
3448 *
3449 * Create a temporary file.
3450 *
3451 * If the inode has been found through an idmapped mount the user namespace of
3452 * the vfsmount must be passed through @mnt_userns. This function will then take
3453 * care to map the inode according to @mnt_userns before checking permissions.
3454 * On non-idmapped mounts or if permission checking is to be performed on the
3455 * raw inode simply passs init_user_ns.
3456 */
3457struct dentry *vfs_tmpfile(struct user_namespace *mnt_userns,
3458			   struct dentry *dentry, umode_t mode, int open_flag)
3459{
3460	struct dentry *child = NULL;
3461	struct inode *dir = dentry->d_inode;
3462	struct inode *inode;
3463	int error;
3464
3465	/* we want directory to be writable */
3466	error = inode_permission(mnt_userns, dir, MAY_WRITE | MAY_EXEC);
3467	if (error)
3468		goto out_err;
3469	error = -EOPNOTSUPP;
3470	if (!dir->i_op->tmpfile)
3471		goto out_err;
3472	error = -ENOMEM;
3473	child = d_alloc(dentry, &slash_name);
3474	if (unlikely(!child))
3475		goto out_err;
3476	error = dir->i_op->tmpfile(mnt_userns, dir, child, mode);
3477	if (error)
3478		goto out_err;
3479	error = -ENOENT;
3480	inode = child->d_inode;
3481	if (unlikely(!inode))
3482		goto out_err;
3483	if (!(open_flag & O_EXCL)) {
3484		spin_lock(&inode->i_lock);
3485		inode->i_state |= I_LINKABLE;
3486		spin_unlock(&inode->i_lock);
3487	}
3488	ima_post_create_tmpfile(mnt_userns, inode);
3489	return child;
3490
3491out_err:
3492	dput(child);
3493	return ERR_PTR(error);
3494}
3495EXPORT_SYMBOL(vfs_tmpfile);
3496
3497static int do_tmpfile(struct nameidata *nd, unsigned flags,
3498		const struct open_flags *op,
3499		struct file *file)
3500{
3501	struct user_namespace *mnt_userns;
3502	struct dentry *child;
3503	struct path path;
3504	int error = path_lookupat(nd, flags | LOOKUP_DIRECTORY, &path);
3505	if (unlikely(error))
3506		return error;
3507	error = mnt_want_write(path.mnt);
3508	if (unlikely(error))
3509		goto out;
3510	mnt_userns = mnt_user_ns(path.mnt);
3511	child = vfs_tmpfile(mnt_userns, path.dentry, op->mode, op->open_flag);
3512	error = PTR_ERR(child);
3513	if (IS_ERR(child))
3514		goto out2;
3515	dput(path.dentry);
3516	path.dentry = child;
3517	audit_inode(nd->name, child, 0);
3518	/* Don't check for other permissions, the inode was just created */
3519	error = may_open(mnt_userns, &path, 0, op->open_flag);
3520	if (!error)
3521		error = vfs_open(&path, file);
3522out2:
3523	mnt_drop_write(path.mnt);
3524out:
3525	path_put(&path);
3526	return error;
3527}
3528
3529static int do_o_path(struct nameidata *nd, unsigned flags, struct file *file)
3530{
3531	struct path path;
3532	int error = path_lookupat(nd, flags, &path);
3533	if (!error) {
3534		audit_inode(nd->name, path.dentry, 0);
3535		error = vfs_open(&path, file);
3536		path_put(&path);
3537	}
3538	return error;
3539}
3540
3541static struct file *path_openat(struct nameidata *nd,
3542			const struct open_flags *op, unsigned flags)
3543{
3544	struct file *file;
3545	int error;
3546
3547	file = alloc_empty_file(op->open_flag, current_cred());
3548	if (IS_ERR(file))
3549		return file;
3550
3551	if (unlikely(file->f_flags & __O_TMPFILE)) {
3552		error = do_tmpfile(nd, flags, op, file);
3553	} else if (unlikely(file->f_flags & O_PATH)) {
3554		error = do_o_path(nd, flags, file);
3555	} else {
3556		const char *s = path_init(nd, flags);
3557		while (!(error = link_path_walk(s, nd)) &&
3558		       (s = open_last_lookups(nd, file, op)) != NULL)
3559			;
3560		if (!error)
3561			error = do_open(nd, file, op);
3562		terminate_walk(nd);
3563	}
3564	if (likely(!error)) {
3565		if (likely(file->f_mode & FMODE_OPENED))
3566			return file;
3567		WARN_ON(1);
3568		error = -EINVAL;
3569	}
3570	fput(file);
3571	if (error == -EOPENSTALE) {
3572		if (flags & LOOKUP_RCU)
3573			error = -ECHILD;
3574		else
3575			error = -ESTALE;
3576	}
3577	return ERR_PTR(error);
3578}
3579
3580struct file *do_filp_open(int dfd, struct filename *pathname,
3581		const struct open_flags *op)
3582{
3583	struct nameidata nd;
3584	int flags = op->lookup_flags;
3585	struct file *filp;
3586
3587	set_nameidata(&nd, dfd, pathname, NULL);
3588	filp = path_openat(&nd, op, flags | LOOKUP_RCU);
3589	if (unlikely(filp == ERR_PTR(-ECHILD)))
3590		filp = path_openat(&nd, op, flags);
3591	if (unlikely(filp == ERR_PTR(-ESTALE)))
3592		filp = path_openat(&nd, op, flags | LOOKUP_REVAL);
3593	restore_nameidata();
3594	return filp;
3595}
3596
3597struct file *do_file_open_root(const struct path *root,
3598		const char *name, const struct open_flags *op)
3599{
3600	struct nameidata nd;
3601	struct file *file;
3602	struct filename *filename;
3603	int flags = op->lookup_flags;
3604
3605	if (d_is_symlink(root->dentry) && op->intent & LOOKUP_OPEN)
3606		return ERR_PTR(-ELOOP);
3607
3608	filename = getname_kernel(name);
3609	if (IS_ERR(filename))
3610		return ERR_CAST(filename);
3611
3612	set_nameidata(&nd, -1, filename, root);
3613	file = path_openat(&nd, op, flags | LOOKUP_RCU);
3614	if (unlikely(file == ERR_PTR(-ECHILD)))
3615		file = path_openat(&nd, op, flags);
3616	if (unlikely(file == ERR_PTR(-ESTALE)))
3617		file = path_openat(&nd, op, flags | LOOKUP_REVAL);
3618	restore_nameidata();
3619	putname(filename);
3620	return file;
3621}
3622
3623static struct dentry *__filename_create(int dfd, struct filename *name,
3624				struct path *path, unsigned int lookup_flags)
3625{
3626	struct dentry *dentry = ERR_PTR(-EEXIST);
3627	struct qstr last;
3628	int type;
3629	int err2;
3630	int error;
3631	bool is_dir = (lookup_flags & LOOKUP_DIRECTORY);
3632
3633	/*
3634	 * Note that only LOOKUP_REVAL and LOOKUP_DIRECTORY matter here. Any
3635	 * other flags passed in are ignored!
3636	 */
3637	lookup_flags &= LOOKUP_REVAL;
3638
3639	error = filename_parentat(dfd, name, lookup_flags, path, &last, &type);
3640	if (error)
3641		return ERR_PTR(error);
3642
3643	/*
3644	 * Yucky last component or no last component at all?
3645	 * (foo/., foo/.., /////)
3646	 */
3647	if (unlikely(type != LAST_NORM))
3648		goto out;
3649
3650	/* don't fail immediately if it's r/o, at least try to report other errors */
3651	err2 = mnt_want_write(path->mnt);
3652	/*
3653	 * Do the final lookup.
3654	 */
3655	lookup_flags |= LOOKUP_CREATE | LOOKUP_EXCL;
3656	inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
3657	dentry = __lookup_hash(&last, path->dentry, lookup_flags);
3658	if (IS_ERR(dentry))
3659		goto unlock;
3660
3661	error = -EEXIST;
3662	if (d_is_positive(dentry))
3663		goto fail;
3664
3665	/*
3666	 * Special case - lookup gave negative, but... we had foo/bar/
3667	 * From the vfs_mknod() POV we just have a negative dentry -
3668	 * all is fine. Let's be bastards - you had / on the end, you've
3669	 * been asking for (non-existent) directory. -ENOENT for you.
3670	 */
3671	if (unlikely(!is_dir && last.name[last.len])) {
3672		error = -ENOENT;
3673		goto fail;
3674	}
3675	if (unlikely(err2)) {
3676		error = err2;
3677		goto fail;
3678	}
3679	return dentry;
3680fail:
3681	dput(dentry);
3682	dentry = ERR_PTR(error);
3683unlock:
3684	inode_unlock(path->dentry->d_inode);
3685	if (!err2)
3686		mnt_drop_write(path->mnt);
3687out:
3688	path_put(path);
3689	return dentry;
3690}
3691
3692static inline struct dentry *filename_create(int dfd, struct filename *name,
3693				struct path *path, unsigned int lookup_flags)
3694{
3695	struct dentry *res = __filename_create(dfd, name, path, lookup_flags);
3696
3697	putname(name);
3698	return res;
3699}
3700
3701struct dentry *kern_path_create(int dfd, const char *pathname,
3702				struct path *path, unsigned int lookup_flags)
3703{
3704	return filename_create(dfd, getname_kernel(pathname),
3705				path, lookup_flags);
3706}
3707EXPORT_SYMBOL(kern_path_create);
3708
3709void done_path_create(struct path *path, struct dentry *dentry)
3710{
3711	dput(dentry);
3712	inode_unlock(path->dentry->d_inode);
3713	mnt_drop_write(path->mnt);
3714	path_put(path);
3715}
3716EXPORT_SYMBOL(done_path_create);
3717
3718inline struct dentry *user_path_create(int dfd, const char __user *pathname,
3719				struct path *path, unsigned int lookup_flags)
3720{
3721	return filename_create(dfd, getname(pathname), path, lookup_flags);
3722}
3723EXPORT_SYMBOL(user_path_create);
3724
3725/**
3726 * vfs_mknod - create device node or file
3727 * @mnt_userns:	user namespace of the mount the inode was found from
3728 * @dir:	inode of @dentry
3729 * @dentry:	pointer to dentry of the base directory
3730 * @mode:	mode of the new device node or file
3731 * @dev:	device number of device to create
3732 *
3733 * Create a device node or file.
3734 *
3735 * If the inode has been found through an idmapped mount the user namespace of
3736 * the vfsmount must be passed through @mnt_userns. This function will then take
3737 * care to map the inode according to @mnt_userns before checking permissions.
3738 * On non-idmapped mounts or if permission checking is to be performed on the
3739 * raw inode simply passs init_user_ns.
3740 */
3741int vfs_mknod(struct user_namespace *mnt_userns, struct inode *dir,
3742	      struct dentry *dentry, umode_t mode, dev_t dev)
3743{
3744	bool is_whiteout = S_ISCHR(mode) && dev == WHITEOUT_DEV;
3745	int error = may_create(mnt_userns, dir, dentry);
3746
3747	if (error)
3748		return error;
3749
3750	if ((S_ISCHR(mode) || S_ISBLK(mode)) && !is_whiteout &&
3751	    !capable(CAP_MKNOD))
3752		return -EPERM;
3753
3754	if (!dir->i_op->mknod)
3755		return -EPERM;
3756
3757	error = devcgroup_inode_mknod(mode, dev);
3758	if (error)
3759		return error;
3760
3761	error = security_inode_mknod(dir, dentry, mode, dev);
3762	if (error)
3763		return error;
3764
3765	error = dir->i_op->mknod(mnt_userns, dir, dentry, mode, dev);
3766	if (!error)
3767		fsnotify_create(dir, dentry);
3768	return error;
3769}
3770EXPORT_SYMBOL(vfs_mknod);
3771
3772static int may_mknod(umode_t mode)
3773{
3774	switch (mode & S_IFMT) {
3775	case S_IFREG:
3776	case S_IFCHR:
3777	case S_IFBLK:
3778	case S_IFIFO:
3779	case S_IFSOCK:
3780	case 0: /* zero mode translates to S_IFREG */
3781		return 0;
3782	case S_IFDIR:
3783		return -EPERM;
3784	default:
3785		return -EINVAL;
3786	}
3787}
3788
3789static int do_mknodat(int dfd, struct filename *name, umode_t mode,
3790		unsigned int dev)
3791{
3792	struct user_namespace *mnt_userns;
3793	struct dentry *dentry;
3794	struct path path;
3795	int error;
3796	unsigned int lookup_flags = 0;
3797
3798	error = may_mknod(mode);
3799	if (error)
3800		goto out1;
3801retry:
3802	dentry = __filename_create(dfd, name, &path, lookup_flags);
3803	error = PTR_ERR(dentry);
3804	if (IS_ERR(dentry))
3805		goto out1;
3806
3807	if (!IS_POSIXACL(path.dentry->d_inode))
3808		mode &= ~current_umask();
3809	error = security_path_mknod(&path, dentry, mode, dev);
3810	if (error)
3811		goto out2;
3812
3813	mnt_userns = mnt_user_ns(path.mnt);
3814	switch (mode & S_IFMT) {
3815		case 0: case S_IFREG:
3816			error = vfs_create(mnt_userns, path.dentry->d_inode,
3817					   dentry, mode, true);
3818			if (!error)
3819				ima_post_path_mknod(mnt_userns, dentry);
3820			break;
3821		case S_IFCHR: case S_IFBLK:
3822			error = vfs_mknod(mnt_userns, path.dentry->d_inode,
3823					  dentry, mode, new_decode_dev(dev));
3824			break;
3825		case S_IFIFO: case S_IFSOCK:
3826			error = vfs_mknod(mnt_userns, path.dentry->d_inode,
3827					  dentry, mode, 0);
3828			break;
3829	}
3830out2:
3831	done_path_create(&path, dentry);
3832	if (retry_estale(error, lookup_flags)) {
3833		lookup_flags |= LOOKUP_REVAL;
3834		goto retry;
3835	}
3836out1:
3837	putname(name);
3838	return error;
3839}
3840
3841SYSCALL_DEFINE4(mknodat, int, dfd, const char __user *, filename, umode_t, mode,
3842		unsigned int, dev)
3843{
3844	return do_mknodat(dfd, getname(filename), mode, dev);
3845}
3846
3847SYSCALL_DEFINE3(mknod, const char __user *, filename, umode_t, mode, unsigned, dev)
3848{
3849	return do_mknodat(AT_FDCWD, getname(filename), mode, dev);
3850}
3851
3852/**
3853 * vfs_mkdir - create directory
3854 * @mnt_userns:	user namespace of the mount the inode was found from
3855 * @dir:	inode of @dentry
3856 * @dentry:	pointer to dentry of the base directory
3857 * @mode:	mode of the new directory
3858 *
3859 * Create a directory.
3860 *
3861 * If the inode has been found through an idmapped mount the user namespace of
3862 * the vfsmount must be passed through @mnt_userns. This function will then take
3863 * care to map the inode according to @mnt_userns before checking permissions.
3864 * On non-idmapped mounts or if permission checking is to be performed on the
3865 * raw inode simply passs init_user_ns.
3866 */
3867int vfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
3868	      struct dentry *dentry, umode_t mode)
3869{
3870	int error = may_create(mnt_userns, dir, dentry);
3871	unsigned max_links = dir->i_sb->s_max_links;
3872
3873	if (error)
3874		return error;
3875
3876	if (!dir->i_op->mkdir)
3877		return -EPERM;
3878
3879	mode &= (S_IRWXUGO|S_ISVTX);
3880	error = security_inode_mkdir(dir, dentry, mode);
3881	if (error)
3882		return error;
3883
3884	if (max_links && dir->i_nlink >= max_links)
3885		return -EMLINK;
3886
3887	error = dir->i_op->mkdir(mnt_userns, dir, dentry, mode);
3888	if (!error)
3889		fsnotify_mkdir(dir, dentry);
3890	return error;
3891}
3892EXPORT_SYMBOL(vfs_mkdir);
3893
3894int do_mkdirat(int dfd, struct filename *name, umode_t mode)
3895{
3896	struct dentry *dentry;
3897	struct path path;
3898	int error;
3899	unsigned int lookup_flags = LOOKUP_DIRECTORY;
3900
3901retry:
3902	dentry = __filename_create(dfd, name, &path, lookup_flags);
3903	error = PTR_ERR(dentry);
3904	if (IS_ERR(dentry))
3905		goto out_putname;
3906
3907	if (!IS_POSIXACL(path.dentry->d_inode))
3908		mode &= ~current_umask();
3909	error = security_path_mkdir(&path, dentry, mode);
3910	if (!error) {
3911		struct user_namespace *mnt_userns;
3912		mnt_userns = mnt_user_ns(path.mnt);
3913		error = vfs_mkdir(mnt_userns, path.dentry->d_inode, dentry,
3914				  mode);
3915	}
3916	done_path_create(&path, dentry);
3917	if (retry_estale(error, lookup_flags)) {
3918		lookup_flags |= LOOKUP_REVAL;
3919		goto retry;
3920	}
3921out_putname:
3922	putname(name);
3923	return error;
3924}
3925
3926SYSCALL_DEFINE3(mkdirat, int, dfd, const char __user *, pathname, umode_t, mode)
3927{
3928	return do_mkdirat(dfd, getname(pathname), mode);
3929}
3930
3931SYSCALL_DEFINE2(mkdir, const char __user *, pathname, umode_t, mode)
3932{
3933	return do_mkdirat(AT_FDCWD, getname(pathname), mode);
3934}
3935
3936/**
3937 * vfs_rmdir - remove directory
3938 * @mnt_userns:	user namespace of the mount the inode was found from
3939 * @dir:	inode of @dentry
3940 * @dentry:	pointer to dentry of the base directory
3941 *
3942 * Remove a directory.
3943 *
3944 * If the inode has been found through an idmapped mount the user namespace of
3945 * the vfsmount must be passed through @mnt_userns. This function will then take
3946 * care to map the inode according to @mnt_userns before checking permissions.
3947 * On non-idmapped mounts or if permission checking is to be performed on the
3948 * raw inode simply passs init_user_ns.
3949 */
3950int vfs_rmdir(struct user_namespace *mnt_userns, struct inode *dir,
3951		     struct dentry *dentry)
3952{
3953	int error = may_delete(mnt_userns, dir, dentry, 1);
3954
3955	if (error)
3956		return error;
3957
3958	if (!dir->i_op->rmdir)
3959		return -EPERM;
3960
3961	dget(dentry);
3962	inode_lock(dentry->d_inode);
3963
3964	error = -EBUSY;
3965	if (is_local_mountpoint(dentry))
3966		goto out;
3967
3968	error = security_inode_rmdir(dir, dentry);
3969	if (error)
3970		goto out;
3971
3972	error = dir->i_op->rmdir(dir, dentry);
3973	if (error)
3974		goto out;
3975
3976	shrink_dcache_parent(dentry);
3977	dentry->d_inode->i_flags |= S_DEAD;
3978	dont_mount(dentry);
3979	detach_mounts(dentry);
3980	fsnotify_rmdir(dir, dentry);
3981
3982out:
3983	inode_unlock(dentry->d_inode);
3984	dput(dentry);
3985	if (!error)
3986		d_delete(dentry);
3987	return error;
3988}
3989EXPORT_SYMBOL(vfs_rmdir);
3990
3991int do_rmdir(int dfd, struct filename *name)
3992{
3993	struct user_namespace *mnt_userns;
3994	int error;
3995	struct dentry *dentry;
3996	struct path path;
3997	struct qstr last;
3998	int type;
3999	unsigned int lookup_flags = 0;
4000retry:
4001	error = filename_parentat(dfd, name, lookup_flags, &path, &last, &type);
4002	if (error)
4003		goto exit1;
4004
4005	switch (type) {
4006	case LAST_DOTDOT:
4007		error = -ENOTEMPTY;
4008		goto exit2;
4009	case LAST_DOT:
4010		error = -EINVAL;
4011		goto exit2;
4012	case LAST_ROOT:
4013		error = -EBUSY;
4014		goto exit2;
4015	}
4016
4017	error = mnt_want_write(path.mnt);
4018	if (error)
4019		goto exit2;
4020
4021	inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT);
4022	dentry = __lookup_hash(&last, path.dentry, lookup_flags);
4023	error = PTR_ERR(dentry);
4024	if (IS_ERR(dentry))
4025		goto exit3;
4026	if (!dentry->d_inode) {
4027		error = -ENOENT;
4028		goto exit4;
4029	}
4030	error = security_path_rmdir(&path, dentry);
4031	if (error)
4032		goto exit4;
4033	mnt_userns = mnt_user_ns(path.mnt);
4034	error = vfs_rmdir(mnt_userns, path.dentry->d_inode, dentry);
4035exit4:
4036	dput(dentry);
4037exit3:
4038	inode_unlock(path.dentry->d_inode);
4039	mnt_drop_write(path.mnt);
4040exit2:
4041	path_put(&path);
4042	if (retry_estale(error, lookup_flags)) {
4043		lookup_flags |= LOOKUP_REVAL;
4044		goto retry;
4045	}
4046exit1:
4047	putname(name);
4048	return error;
4049}
4050
4051SYSCALL_DEFINE1(rmdir, const char __user *, pathname)
4052{
4053	return do_rmdir(AT_FDCWD, getname(pathname));
4054}
4055
4056/**
4057 * vfs_unlink - unlink a filesystem object
4058 * @mnt_userns:	user namespace of the mount the inode was found from
4059 * @dir:	parent directory
4060 * @dentry:	victim
4061 * @delegated_inode: returns victim inode, if the inode is delegated.
4062 *
4063 * The caller must hold dir->i_mutex.
4064 *
4065 * If vfs_unlink discovers a delegation, it will return -EWOULDBLOCK and
4066 * return a reference to the inode in delegated_inode.  The caller
4067 * should then break the delegation on that inode and retry.  Because
4068 * breaking a delegation may take a long time, the caller should drop
4069 * dir->i_mutex before doing so.
4070 *
4071 * Alternatively, a caller may pass NULL for delegated_inode.  This may
4072 * be appropriate for callers that expect the underlying filesystem not
4073 * to be NFS exported.
4074 *
4075 * If the inode has been found through an idmapped mount the user namespace of
4076 * the vfsmount must be passed through @mnt_userns. This function will then take
4077 * care to map the inode according to @mnt_userns before checking permissions.
4078 * On non-idmapped mounts or if permission checking is to be performed on the
4079 * raw inode simply passs init_user_ns.
4080 */
4081int vfs_unlink(struct user_namespace *mnt_userns, struct inode *dir,
4082	       struct dentry *dentry, struct inode **delegated_inode)
4083{
4084	struct inode *target = dentry->d_inode;
4085	int error = may_delete(mnt_userns, dir, dentry, 0);
4086
4087	if (error)
4088		return error;
4089
4090	if (!dir->i_op->unlink)
4091		return -EPERM;
4092
4093	inode_lock(target);
4094	if (IS_SWAPFILE(target))
4095		error = -EPERM;
4096	else if (is_local_mountpoint(dentry))
4097		error = -EBUSY;
4098	else {
4099		error = security_inode_unlink(dir, dentry);
4100		if (!error) {
4101			error = try_break_deleg(target, delegated_inode);
4102			if (error)
4103				goto out;
4104			error = dir->i_op->unlink(dir, dentry);
4105			if (!error) {
4106				dont_mount(dentry);
4107				detach_mounts(dentry);
4108				fsnotify_unlink(dir, dentry);
4109			}
4110		}
4111	}
4112out:
4113	inode_unlock(target);
4114
4115	/* We don't d_delete() NFS sillyrenamed files--they still exist. */
4116	if (!error && !(dentry->d_flags & DCACHE_NFSFS_RENAMED)) {
4117		fsnotify_link_count(target);
4118		d_delete(dentry);
4119	}
4120
4121	return error;
4122}
4123EXPORT_SYMBOL(vfs_unlink);
4124
4125/*
4126 * Make sure that the actual truncation of the file will occur outside its
4127 * directory's i_mutex.  Truncate can take a long time if there is a lot of
4128 * writeout happening, and we don't want to prevent access to the directory
4129 * while waiting on the I/O.
4130 */
4131int do_unlinkat(int dfd, struct filename *name)
4132{
4133	int error;
4134	struct dentry *dentry;
4135	struct path path;
4136	struct qstr last;
4137	int type;
4138	struct inode *inode = NULL;
4139	struct inode *delegated_inode = NULL;
4140	unsigned int lookup_flags = 0;
4141retry:
4142	error = filename_parentat(dfd, name, lookup_flags, &path, &last, &type);
4143	if (error)
4144		goto exit1;
4145
4146	error = -EISDIR;
4147	if (type != LAST_NORM)
4148		goto exit2;
4149
4150	error = mnt_want_write(path.mnt);
4151	if (error)
4152		goto exit2;
4153retry_deleg:
4154	inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT);
4155	dentry = __lookup_hash(&last, path.dentry, lookup_flags);
4156	error = PTR_ERR(dentry);
4157	if (!IS_ERR(dentry)) {
4158		struct user_namespace *mnt_userns;
4159
4160		/* Why not before? Because we want correct error value */
4161		if (last.name[last.len])
4162			goto slashes;
4163		inode = dentry->d_inode;
4164		if (d_is_negative(dentry))
4165			goto slashes;
4166		ihold(inode);
4167		error = security_path_unlink(&path, dentry);
4168		if (error)
4169			goto exit3;
4170		mnt_userns = mnt_user_ns(path.mnt);
4171		error = vfs_unlink(mnt_userns, path.dentry->d_inode, dentry,
4172				   &delegated_inode);
4173exit3:
4174		dput(dentry);
4175	}
4176	inode_unlock(path.dentry->d_inode);
4177	if (inode)
4178		iput(inode);	/* truncate the inode here */
4179	inode = NULL;
4180	if (delegated_inode) {
4181		error = break_deleg_wait(&delegated_inode);
4182		if (!error)
4183			goto retry_deleg;
4184	}
4185	mnt_drop_write(path.mnt);
4186exit2:
4187	path_put(&path);
4188	if (retry_estale(error, lookup_flags)) {
4189		lookup_flags |= LOOKUP_REVAL;
4190		inode = NULL;
4191		goto retry;
4192	}
4193exit1:
4194	putname(name);
4195	return error;
4196
4197slashes:
4198	if (d_is_negative(dentry))
4199		error = -ENOENT;
4200	else if (d_is_dir(dentry))
4201		error = -EISDIR;
4202	else
4203		error = -ENOTDIR;
4204	goto exit3;
4205}
4206
4207SYSCALL_DEFINE3(unlinkat, int, dfd, const char __user *, pathname, int, flag)
4208{
4209	if ((flag & ~AT_REMOVEDIR) != 0)
4210		return -EINVAL;
4211
4212	if (flag & AT_REMOVEDIR)
4213		return do_rmdir(dfd, getname(pathname));
4214	return do_unlinkat(dfd, getname(pathname));
4215}
4216
4217SYSCALL_DEFINE1(unlink, const char __user *, pathname)
4218{
4219	return do_unlinkat(AT_FDCWD, getname(pathname));
4220}
4221
4222/**
4223 * vfs_symlink - create symlink
4224 * @mnt_userns:	user namespace of the mount the inode was found from
4225 * @dir:	inode of @dentry
4226 * @dentry:	pointer to dentry of the base directory
4227 * @oldname:	name of the file to link to
4228 *
4229 * Create a symlink.
4230 *
4231 * If the inode has been found through an idmapped mount the user namespace of
4232 * the vfsmount must be passed through @mnt_userns. This function will then take
4233 * care to map the inode according to @mnt_userns before checking permissions.
4234 * On non-idmapped mounts or if permission checking is to be performed on the
4235 * raw inode simply passs init_user_ns.
4236 */
4237int vfs_symlink(struct user_namespace *mnt_userns, struct inode *dir,
4238		struct dentry *dentry, const char *oldname)
4239{
4240	int error = may_create(mnt_userns, dir, dentry);
4241
4242	if (error)
4243		return error;
4244
4245	if (!dir->i_op->symlink)
4246		return -EPERM;
4247
4248	error = security_inode_symlink(dir, dentry, oldname);
4249	if (error)
4250		return error;
4251
4252	error = dir->i_op->symlink(mnt_userns, dir, dentry, oldname);
4253	if (!error)
4254		fsnotify_create(dir, dentry);
4255	return error;
4256}
4257EXPORT_SYMBOL(vfs_symlink);
4258
4259int do_symlinkat(struct filename *from, int newdfd, struct filename *to)
4260{
4261	int error;
4262	struct dentry *dentry;
4263	struct path path;
4264	unsigned int lookup_flags = 0;
4265
4266	if (IS_ERR(from)) {
4267		error = PTR_ERR(from);
4268		goto out_putnames;
4269	}
4270retry:
4271	dentry = __filename_create(newdfd, to, &path, lookup_flags);
4272	error = PTR_ERR(dentry);
4273	if (IS_ERR(dentry))
4274		goto out_putnames;
4275
4276	error = security_path_symlink(&path, dentry, from->name);
4277	if (!error) {
4278		struct user_namespace *mnt_userns;
4279
4280		mnt_userns = mnt_user_ns(path.mnt);
4281		error = vfs_symlink(mnt_userns, path.dentry->d_inode, dentry,
4282				    from->name);
4283	}
4284	done_path_create(&path, dentry);
4285	if (retry_estale(error, lookup_flags)) {
4286		lookup_flags |= LOOKUP_REVAL;
4287		goto retry;
4288	}
4289out_putnames:
4290	putname(to);
4291	putname(from);
4292	return error;
4293}
4294
4295SYSCALL_DEFINE3(symlinkat, const char __user *, oldname,
4296		int, newdfd, const char __user *, newname)
4297{
4298	return do_symlinkat(getname(oldname), newdfd, getname(newname));
4299}
4300
4301SYSCALL_DEFINE2(symlink, const char __user *, oldname, const char __user *, newname)
4302{
4303	return do_symlinkat(getname(oldname), AT_FDCWD, getname(newname));
4304}
4305
4306/**
4307 * vfs_link - create a new link
4308 * @old_dentry:	object to be linked
4309 * @mnt_userns:	the user namespace of the mount
4310 * @dir:	new parent
4311 * @new_dentry:	where to create the new link
4312 * @delegated_inode: returns inode needing a delegation break
4313 *
4314 * The caller must hold dir->i_mutex
4315 *
4316 * If vfs_link discovers a delegation on the to-be-linked file in need
4317 * of breaking, it will return -EWOULDBLOCK and return a reference to the
4318 * inode in delegated_inode.  The caller should then break the delegation
4319 * and retry.  Because breaking a delegation may take a long time, the
4320 * caller should drop the i_mutex before doing so.
4321 *
4322 * Alternatively, a caller may pass NULL for delegated_inode.  This may
4323 * be appropriate for callers that expect the underlying filesystem not
4324 * to be NFS exported.
4325 *
4326 * If the inode has been found through an idmapped mount the user namespace of
4327 * the vfsmount must be passed through @mnt_userns. This function will then take
4328 * care to map the inode according to @mnt_userns before checking permissions.
4329 * On non-idmapped mounts or if permission checking is to be performed on the
4330 * raw inode simply passs init_user_ns.
4331 */
4332int vfs_link(struct dentry *old_dentry, struct user_namespace *mnt_userns,
4333	     struct inode *dir, struct dentry *new_dentry,
4334	     struct inode **delegated_inode)
4335{
4336	struct inode *inode = old_dentry->d_inode;
4337	unsigned max_links = dir->i_sb->s_max_links;
4338	int error;
4339
4340	if (!inode)
4341		return -ENOENT;
4342
4343	error = may_create(mnt_userns, dir, new_dentry);
4344	if (error)
4345		return error;
4346
4347	if (dir->i_sb != inode->i_sb)
4348		return -EXDEV;
4349
4350	/*
4351	 * A link to an append-only or immutable file cannot be created.
4352	 */
4353	if (IS_APPEND(inode) || IS_IMMUTABLE(inode))
4354		return -EPERM;
4355	/*
4356	 * Updating the link count will likely cause i_uid and i_gid to
4357	 * be writen back improperly if their true value is unknown to
4358	 * the vfs.
4359	 */
4360	if (HAS_UNMAPPED_ID(mnt_userns, inode))
4361		return -EPERM;
4362	if (!dir->i_op->link)
4363		return -EPERM;
4364	if (S_ISDIR(inode->i_mode))
4365		return -EPERM;
4366
4367	error = security_inode_link(old_dentry, dir, new_dentry);
4368	if (error)
4369		return error;
4370
4371	inode_lock(inode);
4372	/* Make sure we don't allow creating hardlink to an unlinked file */
4373	if (inode->i_nlink == 0 && !(inode->i_state & I_LINKABLE))
4374		error =  -ENOENT;
4375	else if (max_links && inode->i_nlink >= max_links)
4376		error = -EMLINK;
4377	else {
4378		error = try_break_deleg(inode, delegated_inode);
4379		if (!error)
4380			error = dir->i_op->link(old_dentry, dir, new_dentry);
4381	}
4382
4383	if (!error && (inode->i_state & I_LINKABLE)) {
4384		spin_lock(&inode->i_lock);
4385		inode->i_state &= ~I_LINKABLE;
4386		spin_unlock(&inode->i_lock);
4387	}
4388	inode_unlock(inode);
4389	if (!error)
4390		fsnotify_link(dir, inode, new_dentry);
4391	return error;
4392}
4393EXPORT_SYMBOL(vfs_link);
4394
4395/*
4396 * Hardlinks are often used in delicate situations.  We avoid
4397 * security-related surprises by not following symlinks on the
4398 * newname.  --KAB
4399 *
4400 * We don't follow them on the oldname either to be compatible
4401 * with linux 2.0, and to avoid hard-linking to directories
4402 * and other special files.  --ADM
4403 */
4404int do_linkat(int olddfd, struct filename *old, int newdfd,
4405	      struct filename *new, int flags)
4406{
4407	struct user_namespace *mnt_userns;
4408	struct dentry *new_dentry;
4409	struct path old_path, new_path;
4410	struct inode *delegated_inode = NULL;
4411	int how = 0;
4412	int error;
4413
4414	if ((flags & ~(AT_SYMLINK_FOLLOW | AT_EMPTY_PATH)) != 0) {
4415		error = -EINVAL;
4416		goto out_putnames;
4417	}
4418	/*
4419	 * To use null names we require CAP_DAC_READ_SEARCH
4420	 * This ensures that not everyone will be able to create
4421	 * handlink using the passed filedescriptor.
4422	 */
4423	if (flags & AT_EMPTY_PATH && !capable(CAP_DAC_READ_SEARCH)) {
4424		error = -ENOENT;
4425		goto out_putnames;
4426	}
4427
4428	if (flags & AT_SYMLINK_FOLLOW)
4429		how |= LOOKUP_FOLLOW;
4430retry:
4431	error = filename_lookup(olddfd, old, how, &old_path, NULL);
4432	if (error)
4433		goto out_putnames;
4434
4435	new_dentry = __filename_create(newdfd, new, &new_path,
4436					(how & LOOKUP_REVAL));
4437	error = PTR_ERR(new_dentry);
4438	if (IS_ERR(new_dentry))
4439		goto out_putpath;
4440
4441	error = -EXDEV;
4442	if (old_path.mnt != new_path.mnt)
4443		goto out_dput;
4444	mnt_userns = mnt_user_ns(new_path.mnt);
4445	error = may_linkat(mnt_userns, &old_path);
4446	if (unlikely(error))
4447		goto out_dput;
4448	error = security_path_link(old_path.dentry, &new_path, new_dentry);
4449	if (error)
4450		goto out_dput;
4451	error = vfs_link(old_path.dentry, mnt_userns, new_path.dentry->d_inode,
4452			 new_dentry, &delegated_inode);
4453out_dput:
4454	done_path_create(&new_path, new_dentry);
4455	if (delegated_inode) {
4456		error = break_deleg_wait(&delegated_inode);
4457		if (!error) {
4458			path_put(&old_path);
4459			goto retry;
4460		}
4461	}
4462	if (retry_estale(error, how)) {
4463		path_put(&old_path);
4464		how |= LOOKUP_REVAL;
4465		goto retry;
4466	}
4467out_putpath:
4468	path_put(&old_path);
4469out_putnames:
4470	putname(old);
4471	putname(new);
4472
4473	return error;
4474}
4475
4476SYSCALL_DEFINE5(linkat, int, olddfd, const char __user *, oldname,
4477		int, newdfd, const char __user *, newname, int, flags)
4478{
4479	return do_linkat(olddfd, getname_uflags(oldname, flags),
4480		newdfd, getname(newname), flags);
4481}
4482
4483SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname)
4484{
4485	return do_linkat(AT_FDCWD, getname(oldname), AT_FDCWD, getname(newname), 0);
4486}
4487
4488/**
4489 * vfs_rename - rename a filesystem object
4490 * @rd:		pointer to &struct renamedata info
4491 *
4492 * The caller must hold multiple mutexes--see lock_rename()).
4493 *
4494 * If vfs_rename discovers a delegation in need of breaking at either
4495 * the source or destination, it will return -EWOULDBLOCK and return a
4496 * reference to the inode in delegated_inode.  The caller should then
4497 * break the delegation and retry.  Because breaking a delegation may
4498 * take a long time, the caller should drop all locks before doing
4499 * so.
4500 *
4501 * Alternatively, a caller may pass NULL for delegated_inode.  This may
4502 * be appropriate for callers that expect the underlying filesystem not
4503 * to be NFS exported.
4504 *
4505 * The worst of all namespace operations - renaming directory. "Perverted"
4506 * doesn't even start to describe it. Somebody in UCB had a heck of a trip...
4507 * Problems:
4508 *
4509 *	a) we can get into loop creation.
4510 *	b) race potential - two innocent renames can create a loop together.
4511 *	   That's where 4.4 screws up. Current fix: serialization on
4512 *	   sb->s_vfs_rename_mutex. We might be more accurate, but that's another
4513 *	   story.
4514 *	c) we have to lock _four_ objects - parents and victim (if it exists),
4515 *	   and source (if it is not a directory).
4516 *	   And that - after we got ->i_mutex on parents (until then we don't know
4517 *	   whether the target exists).  Solution: try to be smart with locking
4518 *	   order for inodes.  We rely on the fact that tree topology may change
4519 *	   only under ->s_vfs_rename_mutex _and_ that parent of the object we
4520 *	   move will be locked.  Thus we can rank directories by the tree
4521 *	   (ancestors first) and rank all non-directories after them.
4522 *	   That works since everybody except rename does "lock parent, lookup,
4523 *	   lock child" and rename is under ->s_vfs_rename_mutex.
4524 *	   HOWEVER, it relies on the assumption that any object with ->lookup()
4525 *	   has no more than 1 dentry.  If "hybrid" objects will ever appear,
4526 *	   we'd better make sure that there's no link(2) for them.
4527 *	d) conversion from fhandle to dentry may come in the wrong moment - when
4528 *	   we are removing the target. Solution: we will have to grab ->i_mutex
4529 *	   in the fhandle_to_dentry code. [FIXME - current nfsfh.c relies on
4530 *	   ->i_mutex on parents, which works but leads to some truly excessive
4531 *	   locking].
4532 */
4533int vfs_rename(struct renamedata *rd)
4534{
4535	int error;
4536	struct inode *old_dir = rd->old_dir, *new_dir = rd->new_dir;
4537	struct dentry *old_dentry = rd->old_dentry;
4538	struct dentry *new_dentry = rd->new_dentry;
4539	struct inode **delegated_inode = rd->delegated_inode;
4540	unsigned int flags = rd->flags;
4541	bool is_dir = d_is_dir(old_dentry);
4542	struct inode *source = old_dentry->d_inode;
4543	struct inode *target = new_dentry->d_inode;
4544	bool new_is_dir = false;
4545	unsigned max_links = new_dir->i_sb->s_max_links;
4546	struct name_snapshot old_name;
4547
4548	if (source == target)
4549		return 0;
4550
4551	error = may_delete(rd->old_mnt_userns, old_dir, old_dentry, is_dir);
4552	if (error)
4553		return error;
4554
4555	if (!target) {
4556		error = may_create(rd->new_mnt_userns, new_dir, new_dentry);
4557	} else {
4558		new_is_dir = d_is_dir(new_dentry);
4559
4560		if (!(flags & RENAME_EXCHANGE))
4561			error = may_delete(rd->new_mnt_userns, new_dir,
4562					   new_dentry, is_dir);
4563		else
4564			error = may_delete(rd->new_mnt_userns, new_dir,
4565					   new_dentry, new_is_dir);
4566	}
4567	if (error)
4568		return error;
4569
4570	if (!old_dir->i_op->rename)
4571		return -EPERM;
4572
4573	/*
4574	 * If we are going to change the parent - check write permissions,
4575	 * we'll need to flip '..'.
4576	 */
4577	if (new_dir != old_dir) {
4578		if (is_dir) {
4579			error = inode_permission(rd->old_mnt_userns, source,
4580						 MAY_WRITE);
4581			if (error)
4582				return error;
4583		}
4584		if ((flags & RENAME_EXCHANGE) && new_is_dir) {
4585			error = inode_permission(rd->new_mnt_userns, target,
4586						 MAY_WRITE);
4587			if (error)
4588				return error;
4589		}
4590	}
4591
4592	error = security_inode_rename(old_dir, old_dentry, new_dir, new_dentry,
4593				      flags);
4594	if (error)
4595		return error;
4596
4597	take_dentry_name_snapshot(&old_name, old_dentry);
4598	dget(new_dentry);
4599	if (!is_dir || (flags & RENAME_EXCHANGE))
4600		lock_two_nondirectories(source, target);
4601	else if (target)
4602		inode_lock(target);
4603
4604	error = -EPERM;
4605	if (IS_SWAPFILE(source) || (target && IS_SWAPFILE(target)))
4606		goto out;
4607
4608	error = -EBUSY;
4609	if (is_local_mountpoint(old_dentry) || is_local_mountpoint(new_dentry))
4610		goto out;
4611
4612	if (max_links && new_dir != old_dir) {
4613		error = -EMLINK;
4614		if (is_dir && !new_is_dir && new_dir->i_nlink >= max_links)
4615			goto out;
4616		if ((flags & RENAME_EXCHANGE) && !is_dir && new_is_dir &&
4617		    old_dir->i_nlink >= max_links)
4618			goto out;
4619	}
4620	if (!is_dir) {
4621		error = try_break_deleg(source, delegated_inode);
4622		if (error)
4623			goto out;
4624	}
4625	if (target && !new_is_dir) {
4626		error = try_break_deleg(target, delegated_inode);
4627		if (error)
4628			goto out;
4629	}
4630	error = old_dir->i_op->rename(rd->new_mnt_userns, old_dir, old_dentry,
4631				      new_dir, new_dentry, flags);
4632	if (error)
4633		goto out;
4634
4635	if (!(flags & RENAME_EXCHANGE) && target) {
4636		if (is_dir) {
4637			shrink_dcache_parent(new_dentry);
4638			target->i_flags |= S_DEAD;
4639		}
4640		dont_mount(new_dentry);
4641		detach_mounts(new_dentry);
4642	}
4643	if (!(old_dir->i_sb->s_type->fs_flags & FS_RENAME_DOES_D_MOVE)) {
4644		if (!(flags & RENAME_EXCHANGE))
4645			d_move(old_dentry, new_dentry);
4646		else
4647			d_exchange(old_dentry, new_dentry);
4648	}
4649out:
4650	if (!is_dir || (flags & RENAME_EXCHANGE))
4651		unlock_two_nondirectories(source, target);
4652	else if (target)
4653		inode_unlock(target);
4654	dput(new_dentry);
4655	if (!error) {
4656		fsnotify_move(old_dir, new_dir, &old_name.name, is_dir,
4657			      !(flags & RENAME_EXCHANGE) ? target : NULL, old_dentry);
4658		if (flags & RENAME_EXCHANGE) {
4659			fsnotify_move(new_dir, old_dir, &old_dentry->d_name,
4660				      new_is_dir, NULL, new_dentry);
4661		}
4662	}
4663	release_dentry_name_snapshot(&old_name);
4664
4665	return error;
4666}
4667EXPORT_SYMBOL(vfs_rename);
4668
4669int do_renameat2(int olddfd, struct filename *from, int newdfd,
4670		 struct filename *to, unsigned int flags)
4671{
4672	struct renamedata rd;
4673	struct dentry *old_dentry, *new_dentry;
4674	struct dentry *trap;
4675	struct path old_path, new_path;
4676	struct qstr old_last, new_last;
4677	int old_type, new_type;
4678	struct inode *delegated_inode = NULL;
4679	unsigned int lookup_flags = 0, target_flags = LOOKUP_RENAME_TARGET;
4680	bool should_retry = false;
4681	int error = -EINVAL;
4682
4683	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
4684		goto put_names;
4685
4686	if ((flags & (RENAME_NOREPLACE | RENAME_WHITEOUT)) &&
4687	    (flags & RENAME_EXCHANGE))
4688		goto put_names;
4689
4690	if (flags & RENAME_EXCHANGE)
4691		target_flags = 0;
4692
4693retry:
4694	error = filename_parentat(olddfd, from, lookup_flags, &old_path,
4695				  &old_last, &old_type);
4696	if (error)
4697		goto put_names;
4698
4699	error = filename_parentat(newdfd, to, lookup_flags, &new_path, &new_last,
4700				  &new_type);
4701	if (error)
4702		goto exit1;
4703
4704	error = -EXDEV;
4705	if (old_path.mnt != new_path.mnt)
4706		goto exit2;
4707
4708	error = -EBUSY;
4709	if (old_type != LAST_NORM)
4710		goto exit2;
4711
4712	if (flags & RENAME_NOREPLACE)
4713		error = -EEXIST;
4714	if (new_type != LAST_NORM)
4715		goto exit2;
4716
4717	error = mnt_want_write(old_path.mnt);
4718	if (error)
4719		goto exit2;
4720
4721retry_deleg:
4722	trap = lock_rename(new_path.dentry, old_path.dentry);
4723
4724	old_dentry = __lookup_hash(&old_last, old_path.dentry, lookup_flags);
4725	error = PTR_ERR(old_dentry);
4726	if (IS_ERR(old_dentry))
4727		goto exit3;
4728	/* source must exist */
4729	error = -ENOENT;
4730	if (d_is_negative(old_dentry))
4731		goto exit4;
4732	new_dentry = __lookup_hash(&new_last, new_path.dentry, lookup_flags | target_flags);
4733	error = PTR_ERR(new_dentry);
4734	if (IS_ERR(new_dentry))
4735		goto exit4;
4736	error = -EEXIST;
4737	if ((flags & RENAME_NOREPLACE) && d_is_positive(new_dentry))
4738		goto exit5;
4739	if (flags & RENAME_EXCHANGE) {
4740		error = -ENOENT;
4741		if (d_is_negative(new_dentry))
4742			goto exit5;
4743
4744		if (!d_is_dir(new_dentry)) {
4745			error = -ENOTDIR;
4746			if (new_last.name[new_last.len])
4747				goto exit5;
4748		}
4749	}
4750	/* unless the source is a directory trailing slashes give -ENOTDIR */
4751	if (!d_is_dir(old_dentry)) {
4752		error = -ENOTDIR;
4753		if (old_last.name[old_last.len])
4754			goto exit5;
4755		if (!(flags & RENAME_EXCHANGE) && new_last.name[new_last.len])
4756			goto exit5;
4757	}
4758	/* source should not be ancestor of target */
4759	error = -EINVAL;
4760	if (old_dentry == trap)
4761		goto exit5;
4762	/* target should not be an ancestor of source */
4763	if (!(flags & RENAME_EXCHANGE))
4764		error = -ENOTEMPTY;
4765	if (new_dentry == trap)
4766		goto exit5;
4767
4768	error = security_path_rename(&old_path, old_dentry,
4769				     &new_path, new_dentry, flags);
4770	if (error)
4771		goto exit5;
4772
4773	rd.old_dir	   = old_path.dentry->d_inode;
4774	rd.old_dentry	   = old_dentry;
4775	rd.old_mnt_userns  = mnt_user_ns(old_path.mnt);
4776	rd.new_dir	   = new_path.dentry->d_inode;
4777	rd.new_dentry	   = new_dentry;
4778	rd.new_mnt_userns  = mnt_user_ns(new_path.mnt);
4779	rd.delegated_inode = &delegated_inode;
4780	rd.flags	   = flags;
4781	error = vfs_rename(&rd);
4782exit5:
4783	dput(new_dentry);
4784exit4:
4785	dput(old_dentry);
4786exit3:
4787	unlock_rename(new_path.dentry, old_path.dentry);
4788	if (delegated_inode) {
4789		error = break_deleg_wait(&delegated_inode);
4790		if (!error)
4791			goto retry_deleg;
4792	}
4793	mnt_drop_write(old_path.mnt);
4794exit2:
4795	if (retry_estale(error, lookup_flags))
4796		should_retry = true;
4797	path_put(&new_path);
4798exit1:
4799	path_put(&old_path);
4800	if (should_retry) {
4801		should_retry = false;
4802		lookup_flags |= LOOKUP_REVAL;
4803		goto retry;
4804	}
4805put_names:
4806	putname(from);
4807	putname(to);
4808	return error;
4809}
4810
4811SYSCALL_DEFINE5(renameat2, int, olddfd, const char __user *, oldname,
4812		int, newdfd, const char __user *, newname, unsigned int, flags)
4813{
4814	return do_renameat2(olddfd, getname(oldname), newdfd, getname(newname),
4815				flags);
4816}
4817
4818SYSCALL_DEFINE4(renameat, int, olddfd, const char __user *, oldname,
4819		int, newdfd, const char __user *, newname)
4820{
4821	return do_renameat2(olddfd, getname(oldname), newdfd, getname(newname),
4822				0);
4823}
4824
4825SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newname)
4826{
4827	return do_renameat2(AT_FDCWD, getname(oldname), AT_FDCWD,
4828				getname(newname), 0);
4829}
4830
4831int readlink_copy(char __user *buffer, int buflen, const char *link)
4832{
4833	int len = PTR_ERR(link);
4834	if (IS_ERR(link))
4835		goto out;
4836
4837	len = strlen(link);
4838	if (len > (unsigned) buflen)
4839		len = buflen;
4840	if (copy_to_user(buffer, link, len))
4841		len = -EFAULT;
4842out:
4843	return len;
4844}
4845
4846/**
4847 * vfs_readlink - copy symlink body into userspace buffer
4848 * @dentry: dentry on which to get symbolic link
4849 * @buffer: user memory pointer
4850 * @buflen: size of buffer
4851 *
4852 * Does not touch atime.  That's up to the caller if necessary
4853 *
4854 * Does not call security hook.
4855 */
4856int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
4857{
4858	struct inode *inode = d_inode(dentry);
4859	DEFINE_DELAYED_CALL(done);
4860	const char *link;
4861	int res;
4862
4863	if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {
4864		if (unlikely(inode->i_op->readlink))
4865			return inode->i_op->readlink(dentry, buffer, buflen);
4866
4867		if (!d_is_symlink(dentry))
4868			return -EINVAL;
4869
4870		spin_lock(&inode->i_lock);
4871		inode->i_opflags |= IOP_DEFAULT_READLINK;
4872		spin_unlock(&inode->i_lock);
4873	}
4874
4875	link = READ_ONCE(inode->i_link);
4876	if (!link) {
4877		link = inode->i_op->get_link(dentry, inode, &done);
4878		if (IS_ERR(link))
4879			return PTR_ERR(link);
4880	}
4881	res = readlink_copy(buffer, buflen, link);
4882	do_delayed_call(&done);
4883	return res;
4884}
4885EXPORT_SYMBOL(vfs_readlink);
4886
4887/**
4888 * vfs_get_link - get symlink body
4889 * @dentry: dentry on which to get symbolic link
4890 * @done: caller needs to free returned data with this
4891 *
4892 * Calls security hook and i_op->get_link() on the supplied inode.
4893 *
4894 * It does not touch atime.  That's up to the caller if necessary.
4895 *
4896 * Does not work on "special" symlinks like /proc/$$/fd/N
4897 */
4898const char *vfs_get_link(struct dentry *dentry, struct delayed_call *done)
4899{
4900	const char *res = ERR_PTR(-EINVAL);
4901	struct inode *inode = d_inode(dentry);
4902
4903	if (d_is_symlink(dentry)) {
4904		res = ERR_PTR(security_inode_readlink(dentry));
4905		if (!res)
4906			res = inode->i_op->get_link(dentry, inode, done);
4907	}
4908	return res;
4909}
4910EXPORT_SYMBOL(vfs_get_link);
4911
4912/* get the link contents into pagecache */
4913const char *page_get_link(struct dentry *dentry, struct inode *inode,
4914			  struct delayed_call *callback)
4915{
4916	char *kaddr;
4917	struct page *page;
4918	struct address_space *mapping = inode->i_mapping;
4919
4920	if (!dentry) {
4921		page = find_get_page(mapping, 0);
4922		if (!page)
4923			return ERR_PTR(-ECHILD);
4924		if (!PageUptodate(page)) {
4925			put_page(page);
4926			return ERR_PTR(-ECHILD);
4927		}
4928	} else {
4929		page = read_mapping_page(mapping, 0, NULL);
4930		if (IS_ERR(page))
4931			return (char*)page;
4932	}
4933	set_delayed_call(callback, page_put_link, page);
4934	BUG_ON(mapping_gfp_mask(mapping) & __GFP_HIGHMEM);
4935	kaddr = page_address(page);
4936	nd_terminate_link(kaddr, inode->i_size, PAGE_SIZE - 1);
4937	return kaddr;
4938}
4939
4940EXPORT_SYMBOL(page_get_link);
4941
4942void page_put_link(void *arg)
4943{
4944	put_page(arg);
4945}
4946EXPORT_SYMBOL(page_put_link);
4947
4948int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)
4949{
4950	DEFINE_DELAYED_CALL(done);
4951	int res = readlink_copy(buffer, buflen,
4952				page_get_link(dentry, d_inode(dentry),
4953					      &done));
4954	do_delayed_call(&done);
4955	return res;
4956}
4957EXPORT_SYMBOL(page_readlink);
4958
4959/*
4960 * The nofs argument instructs pagecache_write_begin to pass AOP_FLAG_NOFS
4961 */
4962int __page_symlink(struct inode *inode, const char *symname, int len, int nofs)
4963{
4964	struct address_space *mapping = inode->i_mapping;
4965	struct page *page;
4966	void *fsdata;
4967	int err;
4968	unsigned int flags = 0;
4969	if (nofs)
4970		flags |= AOP_FLAG_NOFS;
4971
4972retry:
4973	err = pagecache_write_begin(NULL, mapping, 0, len-1,
4974				flags, &page, &fsdata);
4975	if (err)
4976		goto fail;
4977
4978	memcpy(page_address(page), symname, len-1);
4979
4980	err = pagecache_write_end(NULL, mapping, 0, len-1, len-1,
4981							page, fsdata);
4982	if (err < 0)
4983		goto fail;
4984	if (err < len-1)
4985		goto retry;
4986
4987	mark_inode_dirty(inode);
4988	return 0;
4989fail:
4990	return err;
4991}
4992EXPORT_SYMBOL(__page_symlink);
4993
4994int page_symlink(struct inode *inode, const char *symname, int len)
4995{
4996	return __page_symlink(inode, symname, len,
4997			!mapping_gfp_constraint(inode->i_mapping, __GFP_FS));
4998}
4999EXPORT_SYMBOL(page_symlink);
5000
5001const struct inode_operations page_symlink_inode_operations = {
5002	.get_link	= page_get_link,
5003};
5004EXPORT_SYMBOL(page_symlink_inode_operations);
5005