1b2441318SGreg Kroah-Hartman// SPDX-License-Identifier: GPL-2.0
2039363f3SChristoph Lameter/*
3039363f3SChristoph Lameter * Slab allocator functions that are independent of the allocator strategy
4039363f3SChristoph Lameter *
5039363f3SChristoph Lameter * (C) 2012 Christoph Lameter <cl@linux.com>
6039363f3SChristoph Lameter */
7039363f3SChristoph Lameter#include <linux/slab.h>
8039363f3SChristoph Lameter
9039363f3SChristoph Lameter#include <linux/mm.h>
10039363f3SChristoph Lameter#include <linux/poison.h>
11039363f3SChristoph Lameter#include <linux/interrupt.h>
12039363f3SChristoph Lameter#include <linux/memory.h>
131c99ba29SAlexey Dobriyan#include <linux/cache.h>
14039363f3SChristoph Lameter#include <linux/compiler.h>
15d3fb45f3SAlexander Potapenko#include <linux/kfence.h>
16039363f3SChristoph Lameter#include <linux/module.h>
1720cea968SChristoph Lameter#include <linux/cpu.h>
1820cea968SChristoph Lameter#include <linux/uaccess.h>
19b7454ad3SGlauber Costa#include <linux/seq_file.h>
20b7454ad3SGlauber Costa#include <linux/proc_fs.h>
21fcf8a1e4SWaiman Long#include <linux/debugfs.h>
22e86f8b09SAndrey Konovalov#include <linux/kasan.h>
23039363f3SChristoph Lameter#include <asm/cacheflush.h>
24039363f3SChristoph Lameter#include <asm/tlbflush.h>
25039363f3SChristoph Lameter#include <asm/page.h>
262633d7a0SGlauber Costa#include <linux/memcontrol.h>
27928cec9cSAndrey Ryabinin
28928cec9cSAndrey Ryabinin#define CREATE_TRACE_POINTS
29f1b6eb6eSChristoph Lameter#include <trace/events/kmem.h>
30039363f3SChristoph Lameter
3144405099SLong Li#include "internal.h"
3244405099SLong Li
3397d06609SChristoph Lameter#include "slab.h"
3497d06609SChristoph Lameter
3597d06609SChristoph Lameterenum slab_state slab_state;
3618004c5dSChristoph LameterLIST_HEAD(slab_caches);
3718004c5dSChristoph LameterDEFINE_MUTEX(slab_mutex);
389b030cb8SChristoph Lameterstruct kmem_cache *kmem_cache;
3997d06609SChristoph Lameter
40657dc2f9STejun Heostatic LIST_HEAD(slab_caches_to_rcu_destroy);
41657dc2f9STejun Heostatic void slab_caches_to_rcu_destroy_workfn(struct work_struct *work);
42657dc2f9STejun Heostatic DECLARE_WORK(slab_caches_to_rcu_destroy_work,
43657dc2f9STejun Heo		    slab_caches_to_rcu_destroy_workfn);
44657dc2f9STejun Heo
45423c929cSJoonsoo Kim/*
46423c929cSJoonsoo Kim * Set of flags that will prevent slab merging
47423c929cSJoonsoo Kim */
48423c929cSJoonsoo Kim#define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
495f0d5a3aSPaul E. McKenney		SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \
50e86f8b09SAndrey Konovalov		SLAB_FAILSLAB | kasan_never_merge())
51423c929cSJoonsoo Kim
52230e9fc2SVladimir Davydov#define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
536d6ea1e9SNicolas Boichat			 SLAB_CACHE_DMA32 | SLAB_ACCOUNT)
54423c929cSJoonsoo Kim
55423c929cSJoonsoo Kim/*
56423c929cSJoonsoo Kim * Merge control. If this is set then no merging of slab caches will occur.
57423c929cSJoonsoo Kim */
587660a6fdSKees Cookstatic bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT);
59423c929cSJoonsoo Kim
60423c929cSJoonsoo Kimstatic int __init setup_slab_nomerge(char *str)
61423c929cSJoonsoo Kim{
627660a6fdSKees Cook	slab_nomerge = true;
63423c929cSJoonsoo Kim	return 1;
64423c929cSJoonsoo Kim}
65423c929cSJoonsoo Kim
6682edd9d5SRafael Aquinistatic int __init setup_slab_merge(char *str)
6782edd9d5SRafael Aquini{
6882edd9d5SRafael Aquini	slab_nomerge = false;
6982edd9d5SRafael Aquini	return 1;
7082edd9d5SRafael Aquini}
7182edd9d5SRafael Aquini
72423c929cSJoonsoo Kim#ifdef CONFIG_SLUB
73423c929cSJoonsoo Kim__setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0);
7482edd9d5SRafael Aquini__setup_param("slub_merge", slub_merge, setup_slab_merge, 0);
75423c929cSJoonsoo Kim#endif
76423c929cSJoonsoo Kim
77423c929cSJoonsoo Kim__setup("slab_nomerge", setup_slab_nomerge);
7882edd9d5SRafael Aquini__setup("slab_merge", setup_slab_merge);
79423c929cSJoonsoo Kim
8007f361b2SJoonsoo Kim/*
8107f361b2SJoonsoo Kim * Determine the size of a slab object
8207f361b2SJoonsoo Kim */
8307f361b2SJoonsoo Kimunsigned int kmem_cache_size(struct kmem_cache *s)
8407f361b2SJoonsoo Kim{
8507f361b2SJoonsoo Kim	return s->object_size;
8607f361b2SJoonsoo Kim}
8707f361b2SJoonsoo KimEXPORT_SYMBOL(kmem_cache_size);
8807f361b2SJoonsoo Kim
8977be4b13SShuah Khan#ifdef CONFIG_DEBUG_VM
90f4957d5bSAlexey Dobriyanstatic int kmem_cache_sanity_check(const char *name, unsigned int size)
91039363f3SChristoph Lameter{
9274c1d3e0SKees Cook	if (!name || in_interrupt() || size > KMALLOC_MAX_SIZE) {
9377be4b13SShuah Khan		pr_err("kmem_cache_create(%s) integrity check failed\n", name);
9477be4b13SShuah Khan		return -EINVAL;
95039363f3SChristoph Lameter	}
96b920536aSPekka Enberg
9720cea968SChristoph Lameter	WARN_ON(strchr(name, ' '));	/* It confuses parsers */
9877be4b13SShuah Khan	return 0;
9977be4b13SShuah Khan}
10077be4b13SShuah Khan#else
101f4957d5bSAlexey Dobriyanstatic inline int kmem_cache_sanity_check(const char *name, unsigned int size)
10277be4b13SShuah Khan{
10377be4b13SShuah Khan	return 0;
10477be4b13SShuah Khan}
10520cea968SChristoph Lameter#endif
10620cea968SChristoph Lameter
107484748f0SChristoph Lametervoid __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p)
108484748f0SChristoph Lameter{
109484748f0SChristoph Lameter	size_t i;
110484748f0SChristoph Lameter
111ca257195SJesper Dangaard Brouer	for (i = 0; i < nr; i++) {
112ca257195SJesper Dangaard Brouer		if (s)
113ca257195SJesper Dangaard Brouer			kmem_cache_free(s, p[i]);
114ca257195SJesper Dangaard Brouer		else
115ca257195SJesper Dangaard Brouer			kfree(p[i]);
116ca257195SJesper Dangaard Brouer	}
117484748f0SChristoph Lameter}
118484748f0SChristoph Lameter
119865762a8SJesper Dangaard Brouerint __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
120484748f0SChristoph Lameter								void **p)
121484748f0SChristoph Lameter{
122484748f0SChristoph Lameter	size_t i;
123484748f0SChristoph Lameter
124484748f0SChristoph Lameter	for (i = 0; i < nr; i++) {
125484748f0SChristoph Lameter		void *x = p[i] = kmem_cache_alloc(s, flags);
126484748f0SChristoph Lameter		if (!x) {
127484748f0SChristoph Lameter			__kmem_cache_free_bulk(s, i, p);
128865762a8SJesper Dangaard Brouer			return 0;
129484748f0SChristoph Lameter		}
130484748f0SChristoph Lameter	}
131865762a8SJesper Dangaard Brouer	return i;
132484748f0SChristoph Lameter}
133484748f0SChristoph Lameter
134692ae74aSByongho Lee/*
135692ae74aSByongho Lee * Figure out what the alignment of the objects will be given a set of
136692ae74aSByongho Lee * flags, a user specified alignment and the size of the objects.
137692ae74aSByongho Lee */
138f4957d5bSAlexey Dobriyanstatic unsigned int calculate_alignment(slab_flags_t flags,
139f4957d5bSAlexey Dobriyan		unsigned int align, unsigned int size)
140692ae74aSByongho Lee{
141692ae74aSByongho Lee	/*
142692ae74aSByongho Lee	 * If the user wants hardware cache aligned objects then follow that
143692ae74aSByongho Lee	 * suggestion if the object is sufficiently large.
144692ae74aSByongho Lee	 *
145692ae74aSByongho Lee	 * The hardware cache alignment cannot override the specified
146692ae74aSByongho Lee	 * alignment though. If that is greater then use it.
147692ae74aSByongho Lee	 */
148692ae74aSByongho Lee	if (flags & SLAB_HWCACHE_ALIGN) {
149f4957d5bSAlexey Dobriyan		unsigned int ralign;
150692ae74aSByongho Lee
151692ae74aSByongho Lee		ralign = cache_line_size();
152692ae74aSByongho Lee		while (size <= ralign / 2)
153692ae74aSByongho Lee			ralign /= 2;
154692ae74aSByongho Lee		align = max(align, ralign);
155692ae74aSByongho Lee	}
156692ae74aSByongho Lee
157692ae74aSByongho Lee	if (align < ARCH_SLAB_MINALIGN)
158692ae74aSByongho Lee		align = ARCH_SLAB_MINALIGN;
159692ae74aSByongho Lee
160692ae74aSByongho Lee	return ALIGN(align, sizeof(void *));
161692ae74aSByongho Lee}
162692ae74aSByongho Lee
163423c929cSJoonsoo Kim/*
164423c929cSJoonsoo Kim * Find a mergeable slab cache
165423c929cSJoonsoo Kim */
166423c929cSJoonsoo Kimint slab_unmergeable(struct kmem_cache *s)
167423c929cSJoonsoo Kim{
168423c929cSJoonsoo Kim	if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE))
169423c929cSJoonsoo Kim		return 1;
170423c929cSJoonsoo Kim
171423c929cSJoonsoo Kim	if (s->ctor)
172423c929cSJoonsoo Kim		return 1;
173423c929cSJoonsoo Kim
1748eb8284bSDavid Windsor	if (s->usersize)
1758eb8284bSDavid Windsor		return 1;
1768eb8284bSDavid Windsor
177423c929cSJoonsoo Kim	/*
178423c929cSJoonsoo Kim	 * We may have set a slab to be unmergeable during bootstrap.
179423c929cSJoonsoo Kim	 */
180423c929cSJoonsoo Kim	if (s->refcount < 0)
181423c929cSJoonsoo Kim		return 1;
182423c929cSJoonsoo Kim
183423c929cSJoonsoo Kim	return 0;
184423c929cSJoonsoo Kim}
185423c929cSJoonsoo Kim
186f4957d5bSAlexey Dobriyanstruct kmem_cache *find_mergeable(unsigned int size, unsigned int align,
187d50112edSAlexey Dobriyan		slab_flags_t flags, const char *name, void (*ctor)(void *))
188423c929cSJoonsoo Kim{
189423c929cSJoonsoo Kim	struct kmem_cache *s;
190423c929cSJoonsoo Kim
191c6e28895SGrygorii Maistrenko	if (slab_nomerge)
192423c929cSJoonsoo Kim		return NULL;
193423c929cSJoonsoo Kim
194423c929cSJoonsoo Kim	if (ctor)
195423c929cSJoonsoo Kim		return NULL;
196423c929cSJoonsoo Kim
197423c929cSJoonsoo Kim	size = ALIGN(size, sizeof(void *));
198423c929cSJoonsoo Kim	align = calculate_alignment(flags, align, size);
199423c929cSJoonsoo Kim	size = ALIGN(size, align);
20037540008SNikolay Borisov	flags = kmem_cache_flags(size, flags, name);
201423c929cSJoonsoo Kim
202c6e28895SGrygorii Maistrenko	if (flags & SLAB_NEVER_MERGE)
203c6e28895SGrygorii Maistrenko		return NULL;
204c6e28895SGrygorii Maistrenko
205c7094406SRoman Gushchin	list_for_each_entry_reverse(s, &slab_caches, list) {
206423c929cSJoonsoo Kim		if (slab_unmergeable(s))
207423c929cSJoonsoo Kim			continue;
208423c929cSJoonsoo Kim
209423c929cSJoonsoo Kim		if (size > s->size)
210423c929cSJoonsoo Kim			continue;
211423c929cSJoonsoo Kim
212423c929cSJoonsoo Kim		if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME))
213423c929cSJoonsoo Kim			continue;
214423c929cSJoonsoo Kim		/*
215423c929cSJoonsoo Kim		 * Check if alignment is compatible.
216423c929cSJoonsoo Kim		 * Courtesy of Adrian Drzewiecki
217423c929cSJoonsoo Kim		 */
218423c929cSJoonsoo Kim		if ((s->size & ~(align - 1)) != s->size)
219423c929cSJoonsoo Kim			continue;
220423c929cSJoonsoo Kim
221423c929cSJoonsoo Kim		if (s->size - size >= sizeof(void *))
222423c929cSJoonsoo Kim			continue;
223423c929cSJoonsoo Kim
22495069ac8SJoonsoo Kim		if (IS_ENABLED(CONFIG_SLAB) && align &&
22595069ac8SJoonsoo Kim			(align > s->align || s->align % align))
22695069ac8SJoonsoo Kim			continue;
22795069ac8SJoonsoo Kim
228423c929cSJoonsoo Kim		return s;
229423c929cSJoonsoo Kim	}
230423c929cSJoonsoo Kim	return NULL;
231423c929cSJoonsoo Kim}
232423c929cSJoonsoo Kim
233c9a77a79SVladimir Davydovstatic struct kmem_cache *create_cache(const char *name,
234613a5eb5SShakeel Butt		unsigned int object_size, unsigned int align,
2357bbdb81eSAlexey Dobriyan		slab_flags_t flags, unsigned int useroffset,
2367bbdb81eSAlexey Dobriyan		unsigned int usersize, void (*ctor)(void *),
2379855609bSRoman Gushchin		struct kmem_cache *root_cache)
238794b1248SVladimir Davydov{
239794b1248SVladimir Davydov	struct kmem_cache *s;
240794b1248SVladimir Davydov	int err;
241794b1248SVladimir Davydov
2428eb8284bSDavid Windsor	if (WARN_ON(useroffset + usersize > object_size))
2438eb8284bSDavid Windsor		useroffset = usersize = 0;
2448eb8284bSDavid Windsor
245794b1248SVladimir Davydov	err = -ENOMEM;
246794b1248SVladimir Davydov	s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
247794b1248SVladimir Davydov	if (!s)
248794b1248SVladimir Davydov		goto out;
249794b1248SVladimir Davydov
250794b1248SVladimir Davydov	s->name = name;
251613a5eb5SShakeel Butt	s->size = s->object_size = object_size;
252794b1248SVladimir Davydov	s->align = align;
253794b1248SVladimir Davydov	s->ctor = ctor;
2548eb8284bSDavid Windsor	s->useroffset = useroffset;
2558eb8284bSDavid Windsor	s->usersize = usersize;
256794b1248SVladimir Davydov
257794b1248SVladimir Davydov	err = __kmem_cache_create(s, flags);
258794b1248SVladimir Davydov	if (err)
259794b1248SVladimir Davydov		goto out_free_cache;
260794b1248SVladimir Davydov
261794b1248SVladimir Davydov	s->refcount = 1;
262794b1248SVladimir Davydov	list_add(&s->list, &slab_caches);
263794b1248SVladimir Davydovout:
264794b1248SVladimir Davydov	if (err)
265794b1248SVladimir Davydov		return ERR_PTR(err);
266794b1248SVladimir Davydov	return s;
267794b1248SVladimir Davydov
268794b1248SVladimir Davydovout_free_cache:
2697c4da061SVaishali Thakkar	kmem_cache_free(kmem_cache, s);
270794b1248SVladimir Davydov	goto out;
271794b1248SVladimir Davydov}
27245906855SChristoph Lameter
273f496990fSMike Rapoport/**
274f496990fSMike Rapoport * kmem_cache_create_usercopy - Create a cache with a region suitable
275f496990fSMike Rapoport * for copying to userspace
27677be4b13SShuah Khan * @name: A string which is used in /proc/slabinfo to identify this cache.
27777be4b13SShuah Khan * @size: The size of objects to be created in this cache.
27877be4b13SShuah Khan * @align: The required alignment for the objects.
27977be4b13SShuah Khan * @flags: SLAB flags
2808eb8284bSDavid Windsor * @useroffset: Usercopy region offset
281