1b2441318SGreg Kroah-Hartman// SPDX-License-Identifier: GPL-2.0
21da177e4SLinus Torvalds/*
31da177e4SLinus Torvalds * linux/ipc/shm.c
41da177e4SLinus Torvalds * Copyright (C) 1992, 1993 Krishna Balasubramanian
51da177e4SLinus Torvalds *	 Many improvements/fixes by Bruno Haible.
61da177e4SLinus Torvalds * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
71da177e4SLinus Torvalds * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
81da177e4SLinus Torvalds *
91da177e4SLinus Torvalds * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
101da177e4SLinus Torvalds * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
111da177e4SLinus Torvalds * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
121da177e4SLinus Torvalds * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
131da177e4SLinus Torvalds * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
141da177e4SLinus Torvalds * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
151da177e4SLinus Torvalds * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
161da177e4SLinus Torvalds *
17073115d6SSteve Grubb * support for audit of ipc object properties and permission changes
18073115d6SSteve Grubb * Dustin Kirkland <dustin.kirkland@us.ibm.com>
194e982311SKirill Korotaev *
204e982311SKirill Korotaev * namespaces support
214e982311SKirill Korotaev * OpenVZ, SWsoft Inc.
224e982311SKirill Korotaev * Pavel Emelianov <xemul@openvz.org>
23c2c737a0SDavidlohr Bueso *
24c2c737a0SDavidlohr Bueso * Better ipc lock (kern_ipc_perm.lock) handling
25c2c737a0SDavidlohr Bueso * Davidlohr Bueso <davidlohr.bueso@hp.com>, June 2013.
261da177e4SLinus Torvalds */
271da177e4SLinus Torvalds
281da177e4SLinus Torvalds#include <linux/slab.h>
291da177e4SLinus Torvalds#include <linux/mm.h>
301da177e4SLinus Torvalds#include <linux/hugetlb.h>
311da177e4SLinus Torvalds#include <linux/shm.h>
321da177e4SLinus Torvalds#include <linux/init.h>
331da177e4SLinus Torvalds#include <linux/file.h>
341da177e4SLinus Torvalds#include <linux/mman.h>
351da177e4SLinus Torvalds#include <linux/shmem_fs.h>
361da177e4SLinus Torvalds#include <linux/security.h>
371da177e4SLinus Torvalds#include <linux/syscalls.h>
381da177e4SLinus Torvalds#include <linux/audit.h>
39c59ede7bSRandy Dunlap#include <linux/capability.h>
407d87e14cSStephen Rothwell#include <linux/ptrace.h>
4119b4946cSMike Waychison#include <linux/seq_file.h>
423e148c79SNadia Derbey#include <linux/rwsem.h>
434e982311SKirill Korotaev#include <linux/nsproxy.h>
44bc56bba8SEric W. Biederman#include <linux/mount.h>
45ae5e1b22SPavel Emelyanov#include <linux/ipc_namespace.h>
460eb71a9dSNeilBrown#include <linux/rhashtable.h>
477d87e14cSStephen Rothwell
487153e402SPaul McQuade#include <linux/uaccess.h>
491da177e4SLinus Torvalds
501da177e4SLinus Torvalds#include "util.h"
511da177e4SLinus Torvalds
52a2e102cdSEric W. Biedermanstruct shmid_kernel /* private to the kernel */
53a2e102cdSEric W. Biederman{
54a2e102cdSEric W. Biederman	struct kern_ipc_perm	shm_perm;
55a2e102cdSEric W. Biederman	struct file		*shm_file;
56a2e102cdSEric W. Biederman	unsigned long		shm_nattch;
57a2e102cdSEric W. Biederman	unsigned long		shm_segsz;
58a2e102cdSEric W. Biederman	time64_t		shm_atim;
59a2e102cdSEric W. Biederman	time64_t		shm_dtim;
60a2e102cdSEric W. Biederman	time64_t		shm_ctim;
6198f929b1SEric W. Biederman	struct pid		*shm_cprid;
6298f929b1SEric W. Biederman	struct pid		*shm_lprid;
63d7c9e99aSAlexey Gladkov	struct ucounts		*mlock_ucounts;
64a2e102cdSEric W. Biederman
65a2e102cdSEric W. Biederman	/* The task created the shm object.  NULL if the task is dead. */
66a2e102cdSEric W. Biederman	struct task_struct	*shm_creator;
67a2e102cdSEric W. Biederman	struct list_head	shm_clist;	/* list by creator */
68a2e102cdSEric W. Biederman} __randomize_layout;
69a2e102cdSEric W. Biederman
70a2e102cdSEric W. Biederman/* shm_mode upper byte flags */
71a2e102cdSEric W. Biederman#define SHM_DEST	01000	/* segment will be destroyed on last detach */
72a2e102cdSEric W. Biederman#define SHM_LOCKED	02000   /* segment will not be swapped */
73a2e102cdSEric W. Biederman
74bc56bba8SEric W. Biedermanstruct shm_file_data {
75bc56bba8SEric W. Biederman	int id;
76bc56bba8SEric W. Biederman	struct ipc_namespace *ns;
77bc56bba8SEric W. Biederman	struct file *file;
78bc56bba8SEric W. Biederman	const struct vm_operations_struct *vm_ops;
79bc56bba8SEric W. Biederman};
80bc56bba8SEric W. Biederman
81bc56bba8SEric W. Biederman#define shm_file_data(file) (*((struct shm_file_data **)&(file)->private_data))
82bc56bba8SEric W. Biederman
839a32144eSArjan van de Venstatic const struct file_operations shm_file_operations;
84f0f37e2fSAlexey Dobriyanstatic const struct vm_operations_struct shm_vm_ops;
851da177e4SLinus Torvalds
86ed2ddbf8SPierre Peiffer#define shm_ids(ns)	((ns)->ids[IPC_SHM_IDS])
871da177e4SLinus Torvalds
884e982311SKirill Korotaev#define shm_unlock(shp)			\
894e982311SKirill Korotaev	ipc_unlock(&(shp)->shm_perm)
901da177e4SLinus Torvalds
917748dbfaSNadia Derbeystatic int newseg(struct ipc_namespace *, struct ipc_params *);
92bc56bba8SEric W. Biedermanstatic void shm_open(struct vm_area_struct *vma);
93bc56bba8SEric W. Biedermanstatic void shm_close(struct vm_area_struct *vma);
94239521f3SManfred Spraulstatic void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
951da177e4SLinus Torvalds#ifdef CONFIG_PROC_FS
9619b4946cSMike Waychisonstatic int sysvipc_shm_proc_show(struct seq_file *s, void *it);
971da177e4SLinus Torvalds#endif
981da177e4SLinus Torvalds
99eae04d25SDavidlohr Buesovoid shm_init_ns(struct ipc_namespace *ns)
1004e982311SKirill Korotaev{
1014e982311SKirill Korotaev	ns->shm_ctlmax = SHMMAX;
1024e982311SKirill Korotaev	ns->shm_ctlall = SHMALL;
1034e982311SKirill Korotaev	ns->shm_ctlmni = SHMMNI;
104b34a6b1dSVasiliy Kulikov	ns->shm_rmid_forced = 0;
1054e982311SKirill Korotaev	ns->shm_tot = 0;
106eae04d25SDavidlohr Bueso	ipc_init_ids(&shm_ids(ns));
1074e982311SKirill Korotaev}
1084e982311SKirill Korotaev
109f4566f04SNadia Derbey/*
110d9a605e4SDavidlohr Bueso * Called with shm_ids.rwsem (writer) and the shp structure locked.
111d9a605e4SDavidlohr Bueso * Only shm_ids.rwsem remains locked on exit.
112f4566f04SNadia Derbey */
11301b8b07aSPierre Peifferstatic void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
1144e982311SKirill Korotaev{
11501b8b07aSPierre Peiffer	struct shmid_kernel *shp;
11663980c80SShailesh Pandey
11701b8b07aSPierre Peiffer	shp = container_of(ipcp, struct shmid_kernel, shm_perm);
11801b8b07aSPierre Peiffer
119239521f3SManfred Spraul	if (shp->shm_nattch) {
1204e982311SKirill Korotaev		shp->shm_perm.mode |= SHM_DEST;
1214e982311SKirill Korotaev		/* Do not find it any more */
1220cfb6aeeSGuillaume Knispel		ipc_set_key_private(&shm_ids(ns), &shp->shm_perm);
1234e982311SKirill Korotaev		shm_unlock(shp);
1244e982311SKirill Korotaev	} else
1254e982311SKirill Korotaev		shm_destroy(ns, shp);
1264e982311SKirill Korotaev}
1274e982311SKirill Korotaev
128ae5e1b22SPavel Emelyanov#ifdef CONFIG_IPC_NS
1294e982311SKirill Korotaevvoid shm_exit_ns(struct ipc_namespace *ns)
1304e982311SKirill Korotaev{
13101b8b07aSPierre Peiffer	free_ipcs(ns, &shm_ids(ns), do_shm_rmid);
1327d6feeb2SSerge E. Hallyn	idr_destroy(&ns->ids[IPC_SHM_IDS].ipcs_idr);
1330cfb6aeeSGuillaume Knispel	rhashtable_destroy(&ns->ids[IPC_SHM_IDS].key_ht);
1344e982311SKirill Korotaev}
135ae5e1b22SPavel Emelyanov#endif
1361da177e4SLinus Torvalds
137140d0b21SLinus Torvaldsstatic int __init ipc_ns_init(void)
1381da177e4SLinus Torvalds{
139eae04d25SDavidlohr Bueso	shm_init_ns(&init_ipc_ns);
140eae04d25SDavidlohr Bueso	return 0;
141140d0b21SLinus Torvalds}
142140d0b21SLinus Torvalds
143140d0b21SLinus Torvaldspure_initcall(ipc_ns_init);
144140d0b21SLinus Torvalds
145239521f3SManfred Spraulvoid __init shm_init(void)
146140d0b21SLinus Torvalds{
14719b4946cSMike Waychison	ipc_init_proc_interface("sysvipc/shm",
148b7952180SHelge Deller#if BITS_PER_LONG <= 32
149b7952180SHelge Deller				"       key      shmid perms       size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime        rss       swap\n",
150b7952180SHelge Deller#else
151b7952180SHelge Deller				"       key      shmid perms                  size  cpid  lpid nattch   uid   gid  cuid  cgid      atime      dtime      ctime                   rss                  swap\n",
152b7952180SHelge Deller#endif
1534e982311SKirill Korotaev				IPC_SHM_IDS, sysvipc_shm_proc_show);
1541da177e4SLinus Torvalds}
1551da177e4SLinus Torvalds
1568b8d52acSDavidlohr Buesostatic inline struct shmid_kernel *shm_obtain_object(struct ipc_namespace *ns, int id)
1578b8d52acSDavidlohr Bueso{
15855b7ae50SDavidlohr Bueso	struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
1598b8d52acSDavidlohr Bueso
1608b8d52acSDavidlohr Bueso	if (IS_ERR(ipcp))
1618b8d52acSDavidlohr Bueso		return ERR_CAST(ipcp);
1628b8d52acSDavidlohr Bueso
1638b8d52acSDavidlohr Bueso	return container_of(ipcp, struct shmid_kernel, shm_perm);
1648b8d52acSDavidlohr Bueso}
1658b8d52acSDavidlohr Bueso
1668b8d52acSDavidlohr Buesostatic inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace *ns, int id)
1678b8d52acSDavidlohr Bueso{
1688b8d52acSDavidlohr Bueso	struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&shm_ids(ns), id);
1698b8d52acSDavidlohr Bueso
1708b8d52acSDavidlohr Bueso	if (IS_ERR(ipcp))
1718b8d52acSDavidlohr Bueso		return ERR_CAST(ipcp);
1728b8d52acSDavidlohr Bueso
1738b8d52acSDavidlohr Bueso	return container_of(ipcp, struct shmid_kernel, shm_perm);
1748b8d52acSDavidlohr Bueso}
1758b8d52acSDavidlohr Bueso
1763e148c79SNadia Derbey/*
177d9a605e4SDavidlohr Bueso * shm_lock_(check_) routines are called in the paths where the rwsem
17800c2bf85SNadia Derbey * is not necessarily held.
1793e148c79SNadia Derbey */
180023a5355SNadia Derbeystatic inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
1811da177e4SLinus Torvalds{
18282061c57SDavidlohr Bueso	struct kern_ipc_perm *ipcp;
18303f02c76SNadia Derbey
18482061c57SDavidlohr Bueso	rcu_read_lock();
18582061c57SDavidlohr Bueso	ipcp = ipc_obtain_object_idr(&shm_ids(ns), id);
18682061c57SDavidlohr Bueso	if (IS_ERR(ipcp))
18782061c57SDavidlohr Bueso		goto err;
18882061c57SDavidlohr Bueso
18982061c57SDavidlohr Bueso	ipc_lock_object(ipcp);
19082061c57SDavidlohr Bueso	/*
19182061c57SDavidlohr Bueso	 * ipc_rmid() may have already freed the ID while ipc_lock_object()
19282061c57SDavidlohr Bueso	 * was spinning: here verify that the structure is still valid.
19382061c57SDavidlohr Bueso	 * Upon races with RMID, return -EIDRM, thus indicating that
19482061c57SDavidlohr Bueso	 * the ID points to a removed identifier.
19582061c57SDavidlohr Bueso	 */
19682061c57SDavidlohr Bueso	if (ipc_valid_object(ipcp)) {
19782061c57SDavidlohr Bueso		/* return a locked ipc object upon success */
19882061c57SDavidlohr Bueso		return container_of(ipcp, struct shmid_kernel, shm_perm);
19982061c57SDavidlohr Bueso	}
20082061c57SDavidlohr Bueso
20182061c57SDavidlohr Bueso	ipc_unlock_object(ipcp);
2029c21dae2SDavidlohr Bueso	ipcp = ERR_PTR(-EIDRM);
20382061c57SDavidlohr Buesoerr:
20482061c57SDavidlohr Bueso	rcu_read_unlock();
205c5c8975bSDavidlohr Bueso	/*
2061ac0b6deSKirill A. Shutemov	 * Callers of shm_lock() must validate the status of the returned ipc
20782061c57SDavidlohr Bueso	 * object pointer and error out as appropriate.
208c5c8975bSDavidlohr Bueso	 */
20959cf0a93SKees Cook	return ERR_CAST(ipcp);
210023a5355SNadia Derbey}
211023a5355SNadia Derbey
2124c677e2eSVasiliy Kulikovstatic inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
2134c677e2eSVasiliy Kulikov{
2144c677e2eSVasiliy Kulikov	rcu_read_lock();
215cf9d5d78SDavidlohr Bueso	ipc_lock_object(&ipcp->shm_perm);
2164c677e2eSVasiliy Kulikov}
2174c677e2eSVasiliy Kulikov
21853dad6d3SDavidlohr Buesostatic void shm_rcu_free(struct rcu_head *head)
21953dad6d3SDavidlohr Bueso{
220dba4cdd3SManfred Spraul	struct kern_ipc_perm *ptr = container_of(head, struct kern_ipc_perm,
221dba4cdd3SManfred Spraul							rcu);
222dba4cdd3SManfred Spraul	struct shmid_kernel *shp = container_of(ptr, struct shmid_kernel,
223dba4cdd3SManfred Spraul							shm_perm);
2247191adffSEric W. Biederman	security_shm_free(&shp->shm_perm);
225bc8136a5SVasily Averin	kfree(shp);
22653dad6d3SDavidlohr Bueso}
22753dad6d3SDavidlohr Bueso
2287ca7e564SNadia Derbeystatic inline void shm_rmid(struct ipc_namespace *ns, struct shmid_kernel *s)
2291da177e4SLinus Torvalds{
230ab602f79SJack Miller	list_del(&s->shm_clist);
2317ca7e564SNadia Derbey	ipc_rmid(&shm_ids(ns), &s->shm_perm);
2321da177e4SLinus Torvalds}
2331da177e4SLinus Torvalds
2341da177e4SLinus Torvalds
2351ac0b6deSKirill A. Shutemovstatic int __shm_open(struct vm_area_struct *vma)
2364e982311SKirill Korotaev{
237bc56bba8SEric W. Biederman	struct file *file = vma->vm_file;
238bc56bba8SEric W. Biederman	struct shm_file_data *sfd = shm_file_data(file);
2391da177e4SLinus Torvalds	struct shmid_kernel *shp;
2401da177e4SLinus Torvalds
241bc56bba8SEric W. Biederman	shp = shm_lock(sfd->ns, sfd->id);
2421ac0b6deSKirill A. Shutemov
2431ac0b6deSKirill A. Shutemov	if (IS_ERR(shp))
2441ac0b6deSKirill A. Shutemov		return PTR_ERR(shp);
2451ac0b6deSKirill A. Shutemov
2463f05317dSEric Biggers	if (shp->shm_file != sfd->file) {
2473f05317dSEric Biggers		/* ID was reused */
2483f05317dSEric Biggers		shm_unlock(shp);
2493f05317dSEric Biggers		return -EINVAL;
2503f05317dSEric Biggers	}
2513f05317dSEric Biggers
2527ff2819eSDeepa Dinamani	shp->shm_atim = ktime_get_real_seconds();
25398f929b1SEric W. Biederman	ipc_update_pid(&shp->shm_lprid, task_tgid(current));
2541da177e4SLinus Torvalds	shp->shm_nattch++;
2551da177e4SLinus Torvalds	shm_unlock(shp);
2561ac0b6deSKirill A. Shutemov	return 0;
2571ac0b6deSKirill A. Shutemov}
2581ac0b6deSKirill A. Shutemov
2591ac0b6deSKirill A. Shutemov/* This is called by fork, once for every shm attach. */
2601ac0b6deSKirill A. Shutemovstatic void shm_open(struct vm_area_struct *vma)
2611ac0b6deSKirill A. Shutemov{
2621ac0b6deSKirill A. Shutemov	int err = __shm_open(vma);
2631ac0b6deSKirill A. Shutemov	/*
2641ac0b6deSKirill A. Shutemov	 * We raced in the idr lookup or with shm_destroy().
2651ac0b6deSKirill A. Shutemov	 * Either way, the ID is busted.
2661ac0b6deSKirill A. Shutemov	 */
2671ac0b6deSKirill A. Shutemov	WARN_ON_ONCE(err);
2681da177e4SLinus Torvalds}
2691da177e4SLinus Torvalds
2701da177e4SLinus Torvalds/*
2711da177e4SLinus Torvalds * shm_destroy - free the struct shmid_kernel
2721da177e4SLinus Torvalds *
273f4566f04SNadia Derbey * @ns: namespace
2741da177e4SLinus Torvalds * @shp: struct to free
2751da177e4SLinus Torvalds *
276d9a605e4SDavidlohr Bueso * It has to be called with shp and shm_ids.rwsem (writer) locked,
2771da177e4SLinus Torvalds * but returns with shp unlocked and freed.
2781da177e4SLinus Torvalds */
2794e982311SKirill Korotaevstatic void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
2801da177e4SLinus Torvalds{
281a399b29dSGreg Thelen	struct file *shm_file;
282a399b29dSGreg Thelen
283a399b29dSGreg Thelen	shm_file = shp->shm_file;
284a399b29dSGreg Thelen	shp->shm_file = NULL;
2854e982311SKirill Korotaev	ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
2867ca7e564SNadia Derbey	shm_rmid(ns, shp);
2871da177e4SLinus Torvalds	shm_unlock(shp);
288a399b29dSGreg Thelen	if (!is_file_hugepages(shm_file))
289d7c9e99aSAlexey Gladkov		shmem_lock(shm_file, 0, shp->mlock_ucounts);
290d7c9e99aSAlexey Gladkov	else if (shp->mlock_ucounts)
29107a46ed2SDave Hansen		user_shm_unlock(i_size_read(file_inode(shm_file)),
292d7c9e99aSAlexey Gladkov				shp->mlock_ucounts);
293a399b29dSGreg Thelen	fput(shm_file);
29498f929b1SEric W. Biederman	ipc_update_pid(&shp->shm_cprid, NULL);
29598f929b1SEric W. Biederman	ipc_update_pid(&shp->shm_lprid, NULL);
296dba4cdd3SManfred Spraul	ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);
2971da177e4SLinus Torvalds}
2981da177e4SLinus Torvalds
299b34a6b1dSVasiliy Kulikov/*
300b34a6b1dSVasiliy Kulikov * shm_may_destroy - identifies whether shm segment should be destroyed now
301b34a6b1dSVasiliy Kulikov *
302b34a6b1dSVasiliy Kulikov * Returns true if and only if there are no active users of the segment and
303b34a6b1dSVasiliy Kulikov * one of the following is true:
304b34a6b1dSVasiliy Kulikov *
305b34a6b1dSVasiliy Kulikov * 1) shmctl(id, IPC_RMID, NULL) was called for this shp
306b34a6b1dSVasiliy Kulikov *
307b34a6b1dSVasiliy Kulikov * 2) sysctl kernel.shm_rmid_forced is set to 1.
308b34a6b1dSVasiliy Kulikov */
309b34a6b1dSVasiliy Kulikovstatic bool shm_may_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
310b34a6b1dSVasiliy Kulikov{
311b34a6b1dSVasiliy Kulikov	return (shp->shm_nattch == 0) &&
312b34a6b1dSVasiliy Kulikov	       (ns->shm_rmid_forced ||
313b34a6b1dSVasiliy Kulikov		(shp->shm_perm.mode & SHM_DEST));
314