1/* 2 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. 3 * Copyright (C) 2007 The Regents of the University of California. 4 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 5 * Written by Brian Behlendorf <behlendorf1@llnl.gov>. 6 * UCRL-CODE-235197 7 * 8 * This file is part of the SPL, Solaris Porting Layer. 9 * 10 * The SPL is free software; you can redistribute it and/or modify it 11 * under the terms of the GNU General Public License as published by the 12 * Free Software Foundation; either version 2 of the License, or (at your 13 * option) any later version. 14 * 15 * The SPL is distributed in the hope that it will be useful, but WITHOUT 16 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 17 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 18 * for more details. 19 * 20 * You should have received a copy of the GNU General Public License along 21 * with the SPL. If not, see <http://www.gnu.org/licenses/>. 22 */ 23 24#ifndef _SPL_KMEM_H 25#define _SPL_KMEM_H 26 27#include <sys/debug.h> 28#include <linux/slab.h> 29#include <linux/sched.h> 30#include <linux/mm.h> 31#include <linux/vmalloc.h> 32 33extern int kmem_debugging(void); 34extern char *kmem_vasprintf(const char *fmt, va_list ap); 35extern char *kmem_asprintf(const char *fmt, ...); 36extern char *kmem_strdup(const char *str); 37extern void kmem_strfree(char *str); 38 39/* 40 * Memory allocation interfaces 41 */ 42#define KM_SLEEP 0x0000 /* can block for memory; success guaranteed */ 43#define KM_NOSLEEP 0x0001 /* cannot block for memory; may fail */ 44#define KM_PUSHPAGE 0x0004 /* can block for memory; may use reserve */ 45#define KM_ZERO 0x1000 /* zero the allocation */ 46#define KM_VMEM 0x2000 /* caller is vmem_* wrapper */ 47 48#define KM_PUBLIC_MASK (KM_SLEEP | KM_NOSLEEP | KM_PUSHPAGE) 49 50static int spl_fstrans_check(void); 51void *spl_kvmalloc(size_t size, gfp_t flags); 52 53/* 54 * Convert a KM_* flags mask to its Linux GFP_* counterpart. The conversion 55 * function is context aware which means that KM_SLEEP allocations can be 56 * safely used in syncing contexts which have set PF_FSTRANS. 57 */ 58static inline gfp_t 59kmem_flags_convert(int flags) 60{ 61 gfp_t lflags = __GFP_NOWARN | __GFP_COMP; 62 63 if (flags & KM_NOSLEEP) { 64 lflags |= GFP_ATOMIC | __GFP_NORETRY; 65 } else { 66 lflags |= GFP_KERNEL; 67 if (spl_fstrans_check()) 68 lflags &= ~(__GFP_IO|__GFP_FS); 69 } 70 71 if (flags & KM_PUSHPAGE) 72 lflags |= __GFP_HIGH; 73 74 if (flags & KM_ZERO) 75 lflags |= __GFP_ZERO; 76 77 return (lflags); 78} 79 80typedef struct { 81 struct task_struct *fstrans_thread; 82 unsigned int saved_flags; 83} fstrans_cookie_t; 84 85/* 86 * Introduced in Linux 3.9, however this cannot be solely relied on before 87 * Linux 3.18 as it doesn't turn off __GFP_FS as it should. 88 */ 89#ifdef PF_MEMALLOC_NOIO 90#define __SPL_PF_MEMALLOC_NOIO (PF_MEMALLOC_NOIO) 91#else 92#define __SPL_PF_MEMALLOC_NOIO (0) 93#endif 94 95/* 96 * PF_FSTRANS is removed from Linux 4.12 97 */ 98#ifdef PF_FSTRANS 99#define __SPL_PF_FSTRANS (PF_FSTRANS) 100#else 101#define __SPL_PF_FSTRANS (0) 102#endif 103 104#define SPL_FSTRANS (__SPL_PF_FSTRANS|__SPL_PF_MEMALLOC_NOIO) 105 106static inline fstrans_cookie_t 107spl_fstrans_mark(void) 108{ 109 fstrans_cookie_t cookie; 110 111 BUILD_BUG_ON(SPL_FSTRANS == 0); 112 113 cookie.fstrans_thread = current; 114 cookie.saved_flags = current->flags & SPL_FSTRANS; 115 current->flags |= SPL_FSTRANS; 116 117 return (cookie); 118} 119 120static inline void 121spl_fstrans_unmark(fstrans_cookie_t cookie) 122{ 123 ASSERT3P(cookie.fstrans_thread, ==, current); 124 ASSERT((current->flags & SPL_FSTRANS) == SPL_FSTRANS); 125 126 current->flags &= ~SPL_FSTRANS; 127 current->flags |= cookie.saved_flags; 128} 129 130static inline int 131spl_fstrans_check(void) 132{ 133 return (current->flags & SPL_FSTRANS); 134} 135 136/* 137 * specifically used to check PF_FSTRANS flag, cannot be relied on for 138 * checking spl_fstrans_mark(). 139 */ 140static inline int 141__spl_pf_fstrans_check(void) 142{ 143 return (current->flags & __SPL_PF_FSTRANS); 144} 145 146/* 147 * Kernel compatibility for GFP flags 148 */ 149/* < 4.13 */ 150#ifndef __GFP_RETRY_MAYFAIL 151#define __GFP_RETRY_MAYFAIL __GFP_REPEAT 152#endif 153/* < 4.4 */ 154#ifndef __GFP_RECLAIM 155#define __GFP_RECLAIM __GFP_WAIT 156#endif 157 158#ifdef HAVE_ATOMIC64_T 159#define kmem_alloc_used_add(size) atomic64_add(size, &kmem_alloc_used) 160#define kmem_alloc_used_sub(size) atomic64_sub(size, &kmem_alloc_used) 161#define kmem_alloc_used_read() atomic64_read(&kmem_alloc_used) 162#define kmem_alloc_used_set(size) atomic64_set(&kmem_alloc_used, size) 163extern atomic64_t kmem_alloc_used; 164extern unsigned long long kmem_alloc_max; 165#else /* HAVE_ATOMIC64_T */ 166#define kmem_alloc_used_add(size) atomic_add(size, &kmem_alloc_used) 167#define kmem_alloc_used_sub(size) atomic_sub(size, &kmem_alloc_used) 168#define kmem_alloc_used_read() atomic_read(&kmem_alloc_used) 169#define kmem_alloc_used_set(size) atomic_set(&kmem_alloc_used, size) 170extern atomic_t kmem_alloc_used; 171extern unsigned long long kmem_alloc_max; 172#endif /* HAVE_ATOMIC64_T */ 173 174extern unsigned int spl_kmem_alloc_warn; 175extern unsigned int spl_kmem_alloc_max; 176 177#define kmem_alloc(sz, fl) spl_kmem_alloc((sz), (fl), __func__, __LINE__) 178#define kmem_zalloc(sz, fl) spl_kmem_zalloc((sz), (fl), __func__, __LINE__) 179#define kmem_free(ptr, sz) spl_kmem_free((ptr), (sz)) 180#define kmem_cache_reap_active spl_kmem_cache_reap_active 181 182extern void *spl_kmem_alloc(size_t sz, int fl, const char *func, int line); 183extern void *spl_kmem_zalloc(size_t sz, int fl, const char *func, int line); 184extern void spl_kmem_free(const void *ptr, size_t sz); 185 186/* 187 * 5.8 API change, pgprot_t argument removed. 188 */ 189#ifdef HAVE_VMALLOC_PAGE_KERNEL 190#define spl_vmalloc(size, flags) __vmalloc(size, flags, PAGE_KERNEL) 191#else 192#define spl_vmalloc(size, flags) __vmalloc(size, flags) 193#endif 194 195/* 196 * The following functions are only available for internal use. 197 */ 198extern void *spl_kmem_alloc_impl(size_t size, int flags, int node); 199extern void *spl_kmem_alloc_debug(size_t size, int flags, int node); 200extern void *spl_kmem_alloc_track(size_t size, int flags, 201 const char *func, int line, int node); 202extern void spl_kmem_free_impl(const void *buf, size_t size); 203extern void spl_kmem_free_debug(const void *buf, size_t size); 204extern void spl_kmem_free_track(const void *buf, size_t size); 205 206extern int spl_kmem_init(void); 207extern void spl_kmem_fini(void); 208extern int spl_kmem_cache_reap_active(void); 209 210#endif /* _SPL_KMEM_H */ 211