1/* SPDX-License-Identifier: GPL-2.0 */ 2 3/* 4 * This header provides generic wrappers for memory access instrumentation that 5 * the compiler cannot emit for: KASAN, KCSAN, KMSAN. 6 */ 7#ifndef _LINUX_INSTRUMENTED_H 8#define _LINUX_INSTRUMENTED_H 9 10#include <linux/compiler.h> 11#include <linux/kasan-checks.h> 12#include <linux/kcsan-checks.h> 13#include <linux/kmsan-checks.h> 14#include <linux/types.h> 15 16/** 17 * instrument_read - instrument regular read access 18 * @v: address of access 19 * @size: size of access 20 * 21 * Instrument a regular read access. The instrumentation should be inserted 22 * before the actual read happens. 23 */ 24static __always_inline void instrument_read(const volatile void *v, size_t size) 25{ 26 kasan_check_read(v, size); 27 kcsan_check_read(v, size); 28} 29 30/** 31 * instrument_write - instrument regular write access 32 * @v: address of access 33 * @size: size of access 34 * 35 * Instrument a regular write access. The instrumentation should be inserted 36 * before the actual write happens. 37 */ 38static __always_inline void instrument_write(const volatile void *v, size_t size) 39{ 40 kasan_check_write(v, size); 41 kcsan_check_write(v, size); 42} 43 44/** 45 * instrument_read_write - instrument regular read-write access 46 * @v: address of access 47 * @size: size of access 48 * 49 * Instrument a regular write access. The instrumentation should be inserted 50 * before the actual write happens. 51 */ 52static __always_inline void instrument_read_write(const volatile void *v, size_t size) 53{ 54 kasan_check_write(v, size); 55 kcsan_check_read_write(v, size); 56} 57 58/** 59 * instrument_atomic_read - instrument atomic read access 60 * @v: address of access 61 * @size: size of access 62 * 63 * Instrument an atomic read access. The instrumentation should be inserted 64 * before the actual read happens. 65 */ 66static __always_inline void instrument_atomic_read(const volatile void *v, size_t size) 67{ 68 kasan_check_read(v, size); 69 kcsan_check_atomic_read(v, size); 70} 71 72/** 73 * instrument_atomic_write - instrument atomic write access 74 * @v: address of access 75 * @size: size of access 76 * 77 * Instrument an atomic write access. The instrumentation should be inserted 78 * before the actual write happens. 79 */ 80static __always_inline void instrument_atomic_write(const volatile void *v, size_t size) 81{ 82 kasan_check_write(v, size); 83 kcsan_check_atomic_write(v, size); 84} 85 86/** 87 * instrument_atomic_read_write - instrument atomic read-write access 88 * @v: address of access 89 * @size: size of access 90 * 91 * Instrument an atomic read-write access. The instrumentation should be 92 * inserted before the actual write happens. 93 */ 94static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size) 95{ 96 kasan_check_write(v, size); 97 kcsan_check_atomic_read_write(v, size); 98} 99 100/** 101 * instrument_copy_to_user - instrument reads of copy_to_user 102 * @to: destination address 103 * @from: source address 104 * @n: number of bytes to copy 105 * 106 * Instrument reads from kernel memory, that are due to copy_to_user (and 107 * variants). The instrumentation must be inserted before the accesses. 108 */ 109static __always_inline void 110instrument_copy_to_user(void __user *to, const void *from, unsigned long n) 111{ 112 kasan_check_read(from, n); 113 kcsan_check_read(from, n); 114 kmsan_copy_to_user(to, from, n, 0); 115} 116 117/** 118 * instrument_copy_from_user_before - add instrumentation before copy_from_user 119 * @to: destination address 120 * @from: source address 121 * @n: number of bytes to copy 122 * 123 * Instrument writes to kernel memory, that are due to copy_from_user (and 124 * variants). The instrumentation should be inserted before the accesses. 125 */ 126static __always_inline void 127instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n) 128{ 129 kasan_check_write(to, n); 130 kcsan_check_write(to, n); 131} 132 133/** 134 * instrument_copy_from_user_after - add instrumentation after copy_from_user 135 * @to: destination address 136 * @from: source address 137 * @n: number of bytes to copy 138 * @left: number of bytes not copied (as returned by copy_from_user) 139 * 140 * Instrument writes to kernel memory, that are due to copy_from_user (and 141 * variants). The instrumentation should be inserted after the accesses. 142 */ 143static __always_inline void 144instrument_copy_from_user_after(const void *to, const void __user *from, 145 unsigned long n, unsigned long left) 146{ 147 kmsan_unpoison_memory(to, n - left); 148} 149 150/** 151 * instrument_get_user() - add instrumentation to get_user()-like macros 152 * @to: destination variable, may not be address-taken 153 * 154 * get_user() and friends are fragile, so it may depend on the implementation 155 * whether the instrumentation happens before or after the data is copied from 156 * the userspace. 157 */ 158#define instrument_get_user(to) \ 159({ \ 160 u64 __tmp = (u64)(to); \ 161 kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \ 162 to = __tmp; \ 163}) 164 165 166/** 167 * instrument_put_user() - add instrumentation to put_user()-like macros 168 * @from: source address 169 * @ptr: userspace pointer to copy to 170 * @size: number of bytes to copy 171 * 172 * put_user() and friends are fragile, so it may depend on the implementation 173 * whether the instrumentation happens before or after the data is copied from 174 * the userspace. 175 */ 176#define instrument_put_user(from, ptr, size) \ 177({ \ 178 kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \ 179}) 180 181#endif /* _LINUX_INSTRUMENTED_H */ 182