1/*
2 *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
3 *  Copyright (C) 2007 The Regents of the University of California.
4 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
5 *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
6 *  UCRL-CODE-235197
7 *
8 *  This file is part of the SPL, Solaris Porting Layer.
9 *
10 *  The SPL is free software; you can redistribute it and/or modify it
11 *  under the terms of the GNU General Public License as published by the
12 *  Free Software Foundation; either version 2 of the License, or (at your
13 *  option) any later version.
14 *
15 *  The SPL is distributed in the hope that it will be useful, but WITHOUT
16 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
18 *  for more details.
19 *
20 *  You should have received a copy of the GNU General Public License along
21 *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
22 */
23
24#ifndef _SPL_KMEM_H
25#define	_SPL_KMEM_H
26
27#include <sys/debug.h>
28#include <linux/slab.h>
29#include <linux/sched.h>
30#include <linux/mm.h>
31#include <linux/vmalloc.h>
32
33extern int kmem_debugging(void);
34extern char *kmem_vasprintf(const char *fmt, va_list ap);
35extern char *kmem_asprintf(const char *fmt, ...);
36extern char *kmem_strdup(const char *str);
37extern void kmem_strfree(char *str);
38
39/*
40 * Memory allocation interfaces
41 */
42#define	KM_SLEEP	0x0000	/* can block for memory; success guaranteed */
43#define	KM_NOSLEEP	0x0001	/* cannot block for memory; may fail */
44#define	KM_PUSHPAGE	0x0004	/* can block for memory; may use reserve */
45#define	KM_ZERO		0x1000	/* zero the allocation */
46#define	KM_VMEM		0x2000	/* caller is vmem_* wrapper */
47
48#define	KM_PUBLIC_MASK	(KM_SLEEP | KM_NOSLEEP | KM_PUSHPAGE)
49
50static int spl_fstrans_check(void);
51void *spl_kvmalloc(size_t size, gfp_t flags);
52
53/*
54 * Convert a KM_* flags mask to its Linux GFP_* counterpart.  The conversion
55 * function is context aware which means that KM_SLEEP allocations can be
56 * safely used in syncing contexts which have set PF_FSTRANS.
57 */
58static inline gfp_t
59kmem_flags_convert(int flags)
60{
61	gfp_t lflags = __GFP_NOWARN | __GFP_COMP;
62
63	if (flags & KM_NOSLEEP) {
64		lflags |= GFP_ATOMIC | __GFP_NORETRY;
65	} else {
66		lflags |= GFP_KERNEL;
67		if (spl_fstrans_check())
68			lflags &= ~(__GFP_IO|__GFP_FS);
69	}
70
71	if (flags & KM_PUSHPAGE)
72		lflags |= __GFP_HIGH;
73
74	if (flags & KM_ZERO)
75		lflags |= __GFP_ZERO;
76
77	return (lflags);
78}
79
80typedef struct {
81	struct task_struct *fstrans_thread;
82	unsigned int saved_flags;
83} fstrans_cookie_t;
84
85/*
86 * Introduced in Linux 3.9, however this cannot be solely relied on before
87 * Linux 3.18 as it doesn't turn off __GFP_FS as it should.
88 */
89#ifdef PF_MEMALLOC_NOIO
90#define	__SPL_PF_MEMALLOC_NOIO (PF_MEMALLOC_NOIO)
91#else
92#define	__SPL_PF_MEMALLOC_NOIO (0)
93#endif
94
95/*
96 * PF_FSTRANS is removed from Linux 4.12
97 */
98#ifdef PF_FSTRANS
99#define	__SPL_PF_FSTRANS (PF_FSTRANS)
100#else
101#define	__SPL_PF_FSTRANS (0)
102#endif
103
104#define	SPL_FSTRANS (__SPL_PF_FSTRANS|__SPL_PF_MEMALLOC_NOIO)
105
106static inline fstrans_cookie_t
107spl_fstrans_mark(void)
108{
109	fstrans_cookie_t cookie;
110
111	BUILD_BUG_ON(SPL_FSTRANS == 0);
112
113	cookie.fstrans_thread = current;
114	cookie.saved_flags = current->flags & SPL_FSTRANS;
115	current->flags |= SPL_FSTRANS;
116
117	return (cookie);
118}
119
120static inline void
121spl_fstrans_unmark(fstrans_cookie_t cookie)
122{
123	ASSERT3P(cookie.fstrans_thread, ==, current);
124	ASSERT((current->flags & SPL_FSTRANS) == SPL_FSTRANS);
125
126	current->flags &= ~SPL_FSTRANS;
127	current->flags |= cookie.saved_flags;
128}
129
130static inline int
131spl_fstrans_check(void)
132{
133	return (current->flags & SPL_FSTRANS);
134}
135
136/*
137 * specifically used to check PF_FSTRANS flag, cannot be relied on for
138 * checking spl_fstrans_mark().
139 */
140static inline int
141__spl_pf_fstrans_check(void)
142{
143	return (current->flags & __SPL_PF_FSTRANS);
144}
145
146/*
147 * Kernel compatibility for GFP flags
148 */
149/* < 4.13 */
150#ifndef __GFP_RETRY_MAYFAIL
151#define	__GFP_RETRY_MAYFAIL	__GFP_REPEAT
152#endif
153/* < 4.4 */
154#ifndef __GFP_RECLAIM
155#define	__GFP_RECLAIM		__GFP_WAIT
156#endif
157
158#ifdef HAVE_ATOMIC64_T
159#define	kmem_alloc_used_add(size)	atomic64_add(size, &kmem_alloc_used)
160#define	kmem_alloc_used_sub(size)	atomic64_sub(size, &kmem_alloc_used)
161#define	kmem_alloc_used_read()		atomic64_read(&kmem_alloc_used)
162#define	kmem_alloc_used_set(size)	atomic64_set(&kmem_alloc_used, size)
163extern atomic64_t kmem_alloc_used;
164extern unsigned long long kmem_alloc_max;
165#else  /* HAVE_ATOMIC64_T */
166#define	kmem_alloc_used_add(size)	atomic_add(size, &kmem_alloc_used)
167#define	kmem_alloc_used_sub(size)	atomic_sub(size, &kmem_alloc_used)
168#define	kmem_alloc_used_read()		atomic_read(&kmem_alloc_used)
169#define	kmem_alloc_used_set(size)	atomic_set(&kmem_alloc_used, size)
170extern atomic_t kmem_alloc_used;
171extern unsigned long long kmem_alloc_max;
172#endif /* HAVE_ATOMIC64_T */
173
174extern unsigned int spl_kmem_alloc_warn;
175extern unsigned int spl_kmem_alloc_max;
176
177#define	kmem_alloc(sz, fl)	spl_kmem_alloc((sz), (fl), __func__, __LINE__)
178#define	kmem_zalloc(sz, fl)	spl_kmem_zalloc((sz), (fl), __func__, __LINE__)
179#define	kmem_free(ptr, sz)	spl_kmem_free((ptr), (sz))
180#define	kmem_cache_reap_active	spl_kmem_cache_reap_active
181
182extern void *spl_kmem_alloc(size_t sz, int fl, const char *func, int line);
183extern void *spl_kmem_zalloc(size_t sz, int fl, const char *func, int line);
184extern void spl_kmem_free(const void *ptr, size_t sz);
185
186/*
187 * 5.8 API change, pgprot_t argument removed.
188 */
189#ifdef HAVE_VMALLOC_PAGE_KERNEL
190#define	spl_vmalloc(size, flags)	__vmalloc(size, flags, PAGE_KERNEL)
191#else
192#define	spl_vmalloc(size, flags)	__vmalloc(size, flags)
193#endif
194
195/*
196 * The following functions are only available for internal use.
197 */
198extern void *spl_kmem_alloc_impl(size_t size, int flags, int node);
199extern void *spl_kmem_alloc_debug(size_t size, int flags, int node);
200extern void *spl_kmem_alloc_track(size_t size, int flags,
201    const char *func, int line, int node);
202extern void spl_kmem_free_impl(const void *buf, size_t size);
203extern void spl_kmem_free_debug(const void *buf, size_t size);
204extern void spl_kmem_free_track(const void *buf, size_t size);
205
206extern int spl_kmem_init(void);
207extern void spl_kmem_fini(void);
208extern int spl_kmem_cache_reap_active(void);
209
210#endif	/* _SPL_KMEM_H */
211