1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2020 ARM Ltd. 4 */ 5#ifndef __ASM_MTE_KASAN_H 6#define __ASM_MTE_KASAN_H 7 8#include <asm/compiler.h> 9#include <asm/cputype.h> 10#include <asm/mte-def.h> 11 12#ifndef __ASSEMBLY__ 13 14#include <linux/types.h> 15 16#ifdef CONFIG_KASAN_HW_TAGS 17 18/* Whether the MTE asynchronous mode is enabled. */ 19DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode); 20 21static inline bool system_uses_mte_async_or_asymm_mode(void) 22{ 23 return static_branch_unlikely(&mte_async_or_asymm_mode); 24} 25 26#else /* CONFIG_KASAN_HW_TAGS */ 27 28static inline bool system_uses_mte_async_or_asymm_mode(void) 29{ 30 return false; 31} 32 33#endif /* CONFIG_KASAN_HW_TAGS */ 34 35#ifdef CONFIG_ARM64_MTE 36 37/* 38 * The Tag Check Flag (TCF) mode for MTE is per EL, hence TCF0 39 * affects EL0 and TCF affects EL1 irrespective of which TTBR is 40 * used. 41 * The kernel accesses TTBR0 usually with LDTR/STTR instructions 42 * when UAO is available, so these would act as EL0 accesses using 43 * TCF0. 44 * However futex.h code uses exclusives which would be executed as 45 * EL1, this can potentially cause a tag check fault even if the 46 * user disables TCF0. 47 * 48 * To address the problem we set the PSTATE.TCO bit in uaccess_enable() 49 * and reset it in uaccess_disable(). 50 * 51 * The Tag check override (TCO) bit disables temporarily the tag checking 52 * preventing the issue. 53 */ 54static inline void mte_disable_tco(void) 55{ 56 asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(0), 57 ARM64_MTE, CONFIG_KASAN_HW_TAGS)); 58} 59 60static inline void mte_enable_tco(void) 61{ 62 asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(1), 63 ARM64_MTE, CONFIG_KASAN_HW_TAGS)); 64} 65 66/* 67 * These functions disable tag checking only if in MTE async mode 68 * since the sync mode generates exceptions synchronously and the 69 * nofault or load_unaligned_zeropad can handle them. 70 */ 71static inline void __mte_disable_tco_async(void) 72{ 73 if (system_uses_mte_async_or_asymm_mode()) 74 mte_disable_tco(); 75} 76 77static inline void __mte_enable_tco_async(void) 78{ 79 if (system_uses_mte_async_or_asymm_mode()) 80 mte_enable_tco(); 81} 82 83/* 84 * These functions are meant to be only used from KASAN runtime through 85 * the arch_*() interface defined in asm/memory.h. 86 * These functions don't include system_supports_mte() checks, 87 * as KASAN only calls them when MTE is supported and enabled. 88 */ 89 90static inline u8 mte_get_ptr_tag(void *ptr) 91{ 92 /* Note: The format of KASAN tags is 0xF<x> */ 93 u8 tag = 0xF0 | (u8)(((u64)(ptr)) >> MTE_TAG_SHIFT); 94 95 return tag; 96} 97 98/* Get allocation tag for the address. */ 99static inline u8 mte_get_mem_tag(void *addr) 100{ 101 asm(__MTE_PREAMBLE "ldg %0, [%0]" 102 : "+r" (addr)); 103 104 return mte_get_ptr_tag(addr); 105} 106 107/* Generate a random tag. */ 108static inline u8 mte_get_random_tag(void) 109{ 110 void *addr; 111 112 asm(__MTE_PREAMBLE "irg %0, %0" 113 : "=r" (addr)); 114 115 return mte_get_ptr_tag(addr); 116} 117 118static inline u64 __stg_post(u64 p) 119{ 120 asm volatile(__MTE_PREAMBLE "stg %0, [%0], #16" 121 : "+r"(p) 122 : 123 : "memory"); 124 return p; 125} 126 127static inline u64 __stzg_post(u64 p) 128{ 129 asm volatile(__MTE_PREAMBLE "stzg %0, [%0], #16" 130 : "+r"(p) 131 : 132 : "memory"); 133 return p; 134} 135 136static inline void __dc_gva(u64 p) 137{ 138 asm volatile(__MTE_PREAMBLE "dc gva, %0" : : "r"(p) : "memory"); 139} 140 141static inline void __dc_gzva(u64 p) 142{ 143 asm volatile(__MTE_PREAMBLE "dc gzva, %0" : : "r"(p) : "memory"); 144} 145 146/* 147 * Assign allocation tags for a region of memory based on the pointer tag. 148 * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and 149 * size must be MTE_GRANULE_SIZE aligned. 150 */ 151static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag, 152 bool init) 153{ 154 u64 curr, mask, dczid, dczid_bs, dczid_dzp, end1, end2, end3; 155 156 /* Read DC G(Z)VA block size from the system register. */ 157 dczid = read_cpuid(DCZID_EL0); 158 dczid_bs = 4ul << (dczid & 0xf); 159 dczid_dzp = (dczid >> 4) & 1; 160 161 curr = (u64)__tag_set(addr, tag); 162 mask = dczid_bs - 1; 163 /* STG/STZG up to the end of the first block. */ 164 end1 = curr | mask; 165 end3 = curr + size; 166 /* DC GVA / GZVA in [end1, end2) */ 167 end2 = end3 & ~mask; 168 169 /* 170 * The following code uses STG on the first DC GVA block even if the 171 * start address is aligned - it appears to be faster than an alignment 172 * check + conditional branch. Also, if the range size is at least 2 DC 173 * GVA blocks, the first two loops can use post-condition to save one 174 * branch each. 175 */ 176#define SET_MEMTAG_RANGE(stg_post, dc_gva) \ 177 do { \ 178 if (!dczid_dzp && size >= 2 * dczid_bs) {\ 179 do { \ 180 curr = stg_post(curr); \ 181 } while (curr < end1); \ 182 \ 183 do { \ 184 dc_gva(curr); \ 185 curr += dczid_bs; \ 186 } while (curr < end2); \ 187 } \ 188 \ 189 while (curr < end3) \ 190 curr = stg_post(curr); \ 191 } while (0) 192 193 if (init) 194 SET_MEMTAG_RANGE(__stzg_post, __dc_gzva); 195 else 196 SET_MEMTAG_RANGE(__stg_post, __dc_gva); 197#undef SET_MEMTAG_RANGE 198} 199 200void mte_enable_kernel_sync(void); 201void mte_enable_kernel_async(void); 202void mte_enable_kernel_asymm(void); 203 204#else /* CONFIG_ARM64_MTE */ 205 206static inline void mte_disable_tco(void) 207{ 208} 209 210static inline void mte_enable_tco(void) 211{ 212} 213 214static inline void __mte_disable_tco_async(void) 215{ 216} 217 218static inline void __mte_enable_tco_async(void) 219{ 220} 221 222static inline u8 mte_get_ptr_tag(void *ptr) 223{ 224 return 0xFF; 225} 226 227static inline u8 mte_get_mem_tag(void *addr) 228{ 229 return 0xFF; 230} 231 232static inline u8 mte_get_random_tag(void) 233{ 234 return 0xFF; 235} 236 237static inline void mte_set_mem_tag_range(void *addr, size_t size, 238 u8 tag, bool init) 239{ 240} 241 242static inline void mte_enable_kernel_sync(void) 243{ 244} 245 246static inline void mte_enable_kernel_async(void) 247{ 248} 249 250static inline void mte_enable_kernel_asymm(void) 251{ 252} 253 254#endif /* CONFIG_ARM64_MTE */ 255 256#endif /* __ASSEMBLY__ */ 257 258#endif /* __ASM_MTE_KASAN_H */ 259