1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * shadow.c - Shadow Variables 4 * 5 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com> 6 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> 7 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com> 8 */ 9 10/** 11 * DOC: Shadow variable API concurrency notes: 12 * 13 * The shadow variable API provides a simple relationship between an 14 * <obj, id> pair and a pointer value. It is the responsibility of the 15 * caller to provide any mutual exclusion required of the shadow data. 16 * 17 * Once a shadow variable is attached to its parent object via the 18 * klp_shadow_*alloc() API calls, it is considered live: any subsequent 19 * call to klp_shadow_get() may then return the shadow variable's data 20 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data 21 * accordingly. 22 * 23 * The klp_shadow_*alloc() API calls may allocate memory for new shadow 24 * variable structures. Their implementation does not call kmalloc 25 * inside any spinlocks, but API callers should pass GFP flags according 26 * to their specific needs. 27 * 28 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against 29 * concurrent klp_shadow_free() and klp_shadow_get() operations. 30 */ 31 32#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 33 34#include <linux/hashtable.h> 35#include <linux/slab.h> 36#include <linux/livepatch.h> 37 38static DEFINE_HASHTABLE(klp_shadow_hash, 12); 39 40/* 41 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and 42 * the shadow variables it references. 43 */ 44static DEFINE_SPINLOCK(klp_shadow_lock); 45 46/** 47 * struct klp_shadow - shadow variable structure 48 * @node: klp_shadow_hash hash table node 49 * @rcu_head: RCU is used to safely free this structure 50 * @obj: pointer to parent object 51 * @id: data identifier 52 * @data: data area 53 */ 54struct klp_shadow { 55 struct hlist_node node; 56 struct rcu_head rcu_head; 57 void *obj; 58 unsigned long id; 59 char data[]; 60}; 61 62/** 63 * klp_shadow_match() - verify a shadow variable matches given <obj, id> 64 * @shadow: shadow variable to match 65 * @obj: pointer to parent object 66 * @id: data identifier 67 * 68 * Return: true if the shadow variable matches. 69 */ 70static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj, 71 unsigned long id) 72{ 73 return shadow->obj == obj && shadow->id == id; 74} 75 76/** 77 * klp_shadow_get() - retrieve a shadow variable data pointer 78 * @obj: pointer to parent object 79 * @id: data identifier 80 * 81 * Return: the shadow variable data element, NULL on failure. 82 */ 83void *klp_shadow_get(void *obj, unsigned long id) 84{ 85 struct klp_shadow *shadow; 86 87 rcu_read_lock(); 88 89 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node, 90 (unsigned long)obj) { 91 92 if (klp_shadow_match(shadow, obj, id)) { 93 rcu_read_unlock(); 94 return shadow->data; 95 } 96 } 97 98 rcu_read_unlock(); 99 100 return NULL; 101} 102EXPORT_SYMBOL_GPL(klp_shadow_get); 103 104static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id, 105 size_t size, gfp_t gfp_flags, 106 klp_shadow_ctor_t ctor, void *ctor_data, 107 bool warn_on_exist) 108{ 109 struct klp_shadow *new_shadow; 110 void *shadow_data; 111 unsigned long flags; 112 113 /* Check if the shadow variable already exists */ 114 shadow_data = klp_shadow_get(obj, id); 115 if (shadow_data) 116 goto exists; 117 118 /* 119 * Allocate a new shadow variable. Fill it with zeroes by default. 120 * More complex setting can be done by @ctor function. But it is 121 * called only when the buffer is really used (under klp_shadow_lock). 122 */ 123 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags); 124 if (!new_shadow) 125 return NULL; 126 127 /* Look for <obj, id> again under the lock */ 128 spin_lock_irqsave(&klp_shadow_lock, flags); 129 shadow_data = klp_shadow_get(obj, id); 130 if (unlikely(shadow_data)) { 131 /* 132 * Shadow variable was found, throw away speculative 133 * allocation. 134 */ 135 spin_unlock_irqrestore(&klp_shadow_lock, flags); 136 kfree(new_shadow); 137 goto exists; 138 } 139 140 new_shadow->obj = obj; 141 new_shadow->id = id; 142 143 if (ctor) { 144 int err; 145 146 err = ctor(obj, new_shadow->data, ctor_data); 147 if (err) { 148 spin_unlock_irqrestore(&klp_shadow_lock, flags); 149 kfree(new_shadow); 150 pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n", 151 obj, id, err); 152 return NULL; 153 } 154 } 155 156 /* No <obj, id> found, so attach the newly allocated one */ 157 hash_add_rcu(klp_shadow_hash, &new_shadow->node, 158 (unsigned long)new_shadow->obj); 159 spin_unlock_irqrestore(&klp_shadow_lock, flags); 160 161 return new_shadow->data; 162 163exists: 164 if (warn_on_exist) { 165 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id); 166 return NULL; 167 } 168 169 return shadow_data; 170} 171 172/** 173 * klp_shadow_alloc() - allocate and add a new shadow variable 174 * @obj: pointer to parent object 175 * @id: data identifier 176 * @size: size of attached data 177 * @gfp_flags: GFP mask for allocation 178 * @ctor: custom constructor to initialize the shadow data (optional) 179 * @ctor_data: pointer to any data needed by @ctor (optional) 180 * 181 * Allocates @size bytes for new shadow variable data using @gfp_flags. 182 * The data are zeroed by default. They are further initialized by @ctor 183 * function if it is not NULL. The new shadow variable is then added 184 * to the global hashtable. 185 * 186 * If an existing <obj, id> shadow variable can be found, this routine will 187 * issue a WARN, exit early and return NULL. 188 * 189 * This function guarantees that the constructor function is called only when 190 * the variable did not exist before. The cost is that @ctor is called 191 * in atomic context under a spin lock. 192 * 193 * Return: the shadow variable data element, NULL on duplicate or 194 * failure. 195 */ 196void *klp_shadow_alloc(void *obj, unsigned long id, 197 size_t size, gfp_t gfp_flags, 198 klp_shadow_ctor_t ctor, void *ctor_data) 199{ 200 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags, 201 ctor, ctor_data, true); 202} 203EXPORT_SYMBOL_GPL(klp_shadow_alloc); 204 205/** 206 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable 207 * @obj: pointer to parent object 208 * @id: data identifier 209 * @size: size of attached data 210 * @gfp_flags: GFP mask for allocation 211 * @ctor: custom constructor to initialize the shadow data (optional) 212 * @ctor_data: pointer to any data needed by @ctor (optional) 213 * 214 * Returns a pointer to existing shadow data if an <obj, id> shadow 215 * variable is already present. Otherwise, it creates a new shadow 216 * variable like klp_shadow_alloc(). 217 * 218 * This function guarantees that only one shadow variable exists with the given 219 * @id for the given @obj. It also guarantees that the constructor function 220 * will be called only when the variable did not exist before. The cost is 221 * that @ctor is called in atomic context under a spin lock. 222 * 223 * Return: the shadow variable data element, NULL on failure. 224 */ 225void *klp_shadow_get_or_alloc(void *obj, unsigned long id, 226 size_t size, gfp_t gfp_flags, 227 klp_shadow_ctor_t ctor, void *ctor_data) 228{ 229 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags, 230 ctor, ctor_data, false); 231} 232EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc); 233 234static void klp_shadow_free_struct(struct klp_shadow *shadow, 235 klp_shadow_dtor_t dtor) 236{ 237 hash_del_rcu(&shadow->node); 238 if (dtor) 239 dtor(shadow->obj, shadow->data); 240 kfree_rcu(shadow, rcu_head); 241} 242 243/** 244 * klp_shadow_free() - detach and free a <obj, id> shadow variable 245 * @obj: pointer to parent object 246 * @id: data identifier 247 * @dtor: custom callback that can be used to unregister the variable 248 * and/or free data that the shadow variable points to (optional) 249 * 250 * This function releases the memory for this <obj, id> shadow variable 251 * instance, callers should stop referencing it accordingly. 252 */ 253void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor) 254{ 255 struct klp_shadow *shadow; 256 unsigned long flags; 257 258 spin_lock_irqsave(&klp_shadow_lock, flags); 259 260 /* Delete <obj, id> from hash */ 261 hash_for_each_possible(klp_shadow_hash, shadow, node, 262 (unsigned long)obj) { 263 264 if (klp_shadow_match(shadow, obj, id)) { 265 klp_shadow_free_struct(shadow, dtor); 266 break; 267 } 268 } 269 270 spin_unlock_irqrestore(&klp_shadow_lock, flags); 271} 272EXPORT_SYMBOL_GPL(klp_shadow_free); 273 274/** 275 * klp_shadow_free_all() - detach and free all <_, id> shadow variables 276 * @id: data identifier 277 * @dtor: custom callback that can be used to unregister the variable 278 * and/or free data that the shadow variable points to (optional) 279 * 280 * This function releases the memory for all <_, id> shadow variable 281 * instances, callers should stop referencing them accordingly. 282 */ 283void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor) 284{ 285 struct klp_shadow *shadow; 286 unsigned long flags; 287 int i; 288 289 spin_lock_irqsave(&klp_shadow_lock, flags); 290 291 /* Delete all <_, id> from hash */ 292 hash_for_each(klp_shadow_hash, i, shadow, node) { 293 if (klp_shadow_match(shadow, shadow->obj, id)) 294 klp_shadow_free_struct(shadow, dtor); 295 } 296 297 spin_unlock_irqrestore(&klp_shadow_lock, flags); 298} 299EXPORT_SYMBOL_GPL(klp_shadow_free_all); 300