1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _ASM_POWERPC_BOOK3S_64_KUP_H 3#define _ASM_POWERPC_BOOK3S_64_KUP_H 4 5#include <linux/const.h> 6#include <asm/reg.h> 7 8#define AMR_KUAP_BLOCK_READ UL(0x5455555555555555) 9#define AMR_KUAP_BLOCK_WRITE UL(0xa8aaaaaaaaaaaaaa) 10#define AMR_KUEP_BLOCKED UL(0x5455555555555555) 11#define AMR_KUAP_BLOCKED (AMR_KUAP_BLOCK_READ | AMR_KUAP_BLOCK_WRITE) 12 13#ifdef __ASSEMBLY__ 14 15.macro kuap_user_restore gpr1, gpr2 16#if defined(CONFIG_PPC_PKEY) 17 BEGIN_MMU_FTR_SECTION_NESTED(67) 18 b 100f // skip_restore_amr 19 END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67) 20 /* 21 * AMR and IAMR are going to be different when 22 * returning to userspace. 23 */ 24 ld \gpr1, STACK_REGS_AMR(r1) 25 26 /* 27 * If kuap feature is not enabled, do the mtspr 28 * only if AMR value is different. 29 */ 30 BEGIN_MMU_FTR_SECTION_NESTED(68) 31 mfspr \gpr2, SPRN_AMR 32 cmpd \gpr1, \gpr2 33 beq 99f 34 END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_KUAP, 68) 35 36 isync 37 mtspr SPRN_AMR, \gpr1 3899: 39 /* 40 * Restore IAMR only when returning to userspace 41 */ 42 ld \gpr1, STACK_REGS_IAMR(r1) 43 44 /* 45 * If kuep feature is not enabled, do the mtspr 46 * only if IAMR value is different. 47 */ 48 BEGIN_MMU_FTR_SECTION_NESTED(69) 49 mfspr \gpr2, SPRN_IAMR 50 cmpd \gpr1, \gpr2 51 beq 100f 52 END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUEP, 69) 53 54 isync 55 mtspr SPRN_IAMR, \gpr1 56 57100: //skip_restore_amr 58 /* No isync required, see kuap_user_restore() */ 59#endif 60.endm 61 62.macro kuap_kernel_restore gpr1, gpr2 63#if defined(CONFIG_PPC_PKEY) 64 65 BEGIN_MMU_FTR_SECTION_NESTED(67) 66 /* 67 * AMR is going to be mostly the same since we are 68 * returning to the kernel. Compare and do a mtspr. 69 */ 70 ld \gpr2, STACK_REGS_AMR(r1) 71 mfspr \gpr1, SPRN_AMR 72 cmpd \gpr1, \gpr2 73 beq 100f 74 isync 75 mtspr SPRN_AMR, \gpr2 76 /* 77 * No isync required, see kuap_restore_amr() 78 * No need to restore IAMR when returning to kernel space. 79 */ 80100: 81 END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 67) 82#endif 83.endm 84 85#ifdef CONFIG_PPC_KUAP 86.macro kuap_check_amr gpr1, gpr2 87#ifdef CONFIG_PPC_KUAP_DEBUG 88 BEGIN_MMU_FTR_SECTION_NESTED(67) 89 mfspr \gpr1, SPRN_AMR 90 /* Prevent access to userspace using any key values */ 91 LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED) 92999: tdne \gpr1, \gpr2 93 EMIT_WARN_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE) 94 END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 67) 95#endif 96.endm 97#endif 98 99/* 100 * if (pkey) { 101 * 102 * save AMR -> stack; 103 * if (kuap) { 104 * if (AMR != BLOCKED) 105 * KUAP_BLOCKED -> AMR; 106 * } 107 * if (from_user) { 108 * save IAMR -> stack; 109 * if (kuep) { 110 * KUEP_BLOCKED ->IAMR 111 * } 112 * } 113 * return; 114 * } 115 * 116 * if (kuap) { 117 * if (from_kernel) { 118 * save AMR -> stack; 119 * if (AMR != BLOCKED) 120 * KUAP_BLOCKED -> AMR; 121 * } 122 * 123 * } 124 */ 125.macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr 126#if defined(CONFIG_PPC_PKEY) 127 128 /* 129 * if both pkey and kuap is disabled, nothing to do 130 */ 131 BEGIN_MMU_FTR_SECTION_NESTED(68) 132 b 100f // skip_save_amr 133 END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY | MMU_FTR_KUAP, 68) 134 135 /* 136 * if pkey is disabled and we are entering from userspace 137 * don't do anything. 138 */ 139 BEGIN_MMU_FTR_SECTION_NESTED(67) 140 .ifnb \msr_pr_cr 141 /* 142 * Without pkey we are not changing AMR outside the kernel 143 * hence skip this completely. 144 */ 145 bne \msr_pr_cr, 100f // from userspace 146 .endif 147 END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67) 148 149 /* 150 * pkey is enabled or pkey is disabled but entering from kernel 151 */ 152 mfspr \gpr1, SPRN_AMR 153 std \gpr1, STACK_REGS_AMR(r1) 154 155 /* 156 * update kernel AMR with AMR_KUAP_BLOCKED only 157 * if KUAP feature is enabled 158 */ 159 BEGIN_MMU_FTR_SECTION_NESTED(69) 160 LOAD_REG_IMMEDIATE(\gpr2, AMR_KUAP_BLOCKED) 161 cmpd \use_cr, \gpr1, \gpr2 162 beq \use_cr, 102f 163 /* 164 * We don't isync here because we very recently entered via an interrupt 165 */ 166 mtspr SPRN_AMR, \gpr2 167 isync 168102: 169 END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_KUAP, 69) 170 171 /* 172 * if entering from kernel we don't need save IAMR 173 */ 174 .ifnb \msr_pr_cr 175 beq \msr_pr_cr, 100f // from kernel space 176 mfspr \gpr1, SPRN_IAMR 177 std \gpr1, STACK_REGS_IAMR(r1) 178 179 /* 180 * update kernel IAMR with AMR_KUEP_BLOCKED only 181 * if KUEP feature is enabled 182 */ 183 BEGIN_MMU_FTR_SECTION_NESTED(70) 184 LOAD_REG_IMMEDIATE(\gpr2, AMR_KUEP_BLOCKED) 185 mtspr SPRN_IAMR, \gpr2 186 isync 187 END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_BOOK3S_KUEP, 70) 188 .endif 189 190100: // skip_save_amr 191#endif 192.endm 193 194#else /* !__ASSEMBLY__ */ 195 196#include <linux/jump_label.h> 197#include <linux/sched.h> 198 199DECLARE_STATIC_KEY_FALSE(uaccess_flush_key); 200 201#ifdef CONFIG_PPC_PKEY 202 203extern u64 __ro_after_init default_uamor; 204extern u64 __ro_after_init default_amr; 205extern u64 __ro_after_init default_iamr; 206 207#include <asm/mmu.h> 208#include <asm/ptrace.h> 209 210/* usage of kthread_use_mm() should inherit the 211 * AMR value of the operating address space. But, the AMR value is 212 * thread-specific and we inherit the address space and not thread 213 * access restrictions. Because of this ignore AMR value when accessing 214 * userspace via kernel thread. 215 */ 216static __always_inline u64 current_thread_amr(void) 217{ 218 if (current->thread.regs) 219 return current->thread.regs->amr; 220 return default_amr; 221} 222 223static __always_inline u64 current_thread_iamr(void) 224{ 225 if (current->thread.regs) 226 return current->thread.regs->iamr; 227 return default_iamr; 228} 229#endif /* CONFIG_PPC_PKEY */ 230 231#ifdef CONFIG_PPC_KUAP 232 233static __always_inline void kuap_user_restore(struct pt_regs *regs) 234{ 235 bool restore_amr = false, restore_iamr = false; 236 unsigned long amr, iamr; 237 238 if (!mmu_has_feature(MMU_FTR_PKEY)) 239 return; 240 241 if (!mmu_has_feature(MMU_FTR_KUAP)) { 242 amr = mfspr(SPRN_AMR); 243 if (amr != regs->amr) 244 restore_amr = true; 245 } else { 246 restore_amr = true; 247 } 248 249 if (!mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) { 250 iamr = mfspr(SPRN_IAMR); 251 if (iamr != regs->iamr) 252 restore_iamr = true; 253 } else { 254 restore_iamr = true; 255 } 256 257 258 if (restore_amr || restore_iamr) { 259 isync(); 260 if (restore_amr) 261 mtspr(SPRN_AMR, regs->amr); 262 if (restore_iamr) 263 mtspr(SPRN_IAMR, regs->iamr); 264 } 265 /* 266 * No isync required here because we are about to rfi 267 * back to previous context before any user accesses 268 * would be made, which is a CSI. 269 */ 270} 271 272static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) 273{ 274 if (likely(regs->amr == amr)) 275 return; 276 277 isync(); 278 mtspr(SPRN_AMR, regs->amr); 279 /* 280 * No isync required here because we are about to rfi 281 * back to previous context before any user accesses 282 * would be made, which is a CSI. 283 * 284 * No need to restore IAMR when returning to kernel space. 285 */ 286} 287 288static __always_inline unsigned long __kuap_get_and_assert_locked(void) 289{ 290 unsigned long amr = mfspr(SPRN_AMR); 291 292 if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG)) /* kuap_check_amr() */ 293 WARN_ON_ONCE(amr != AMR_KUAP_BLOCKED); 294 return amr; 295} 296#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked 297 298/* __kuap_lock() not required, book3s/64 does that in ASM */ 299 300/* 301 * We support individually allowing read or write, but we don't support nesting 302 * because that would require an expensive read/modify write of the AMR. 303 */ 304 305static __always_inline unsigned long get_kuap(void) 306{ 307 /* 308 * We return AMR_KUAP_BLOCKED when we don't support KUAP because 309 * prevent_user_access_return needs to return AMR_KUAP_BLOCKED to 310 * cause restore_user_access to do a flush. 311 * 312 * This has no effect in terms of actually blocking things on hash, 313 * so it doesn't break anything. 314 */ 315 if (!mmu_has_feature(MMU_FTR_KUAP)) 316 return AMR_KUAP_BLOCKED; 317 318 return mfspr(SPRN_AMR); 319} 320 321static __always_inline void set_kuap(unsigned long value) 322{ 323 if (!mmu_has_feature(MMU_FTR_KUAP)) 324 return; 325 326 /* 327 * ISA v3.0B says we need a CSI (Context Synchronising Instruction) both 328 * before and after the move to AMR. See table 6 on page 1134. 329 */ 330 isync(); 331 mtspr(SPRN_AMR, value); 332 isync(); 333} 334 335static __always_inline bool 336__bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) 337{ 338 /* 339 * For radix this will be a storage protection fault (DSISR_PROTFAULT). 340 * For hash this will be a key fault (DSISR_KEYFAULT) 341 */ 342 /* 343 * We do have exception table entry, but accessing the 344 * userspace results in fault. This could be because we 345 * didn't unlock the AMR or access is denied by userspace 346 * using a key value that blocks access. We are only interested 347 * in catching the use case of accessing without unlocking 348 * the AMR. Hence check for BLOCK_WRITE/READ against AMR. 349 */ 350 if (is_write) { 351 return (regs->amr & AMR_KUAP_BLOCK_WRITE) == AMR_KUAP_BLOCK_WRITE; 352 } 353 return (regs->amr & AMR_KUAP_BLOCK_READ) == AMR_KUAP_BLOCK_READ; 354} 355 356static __always_inline void allow_user_access(void __user *to, const void __user *from, 357 unsigned long size, unsigned long dir) 358{ 359 unsigned long thread_amr = 0; 360 361 // This is written so we can resolve to a single case at build time 362 BUILD_BUG_ON(!__builtin_constant_p(dir)); 363 364 if (mmu_has_feature(MMU_FTR_PKEY)) 365 thread_amr = current_thread_amr(); 366 367 if (dir == KUAP_READ) 368 set_kuap(thread_amr | AMR_KUAP_BLOCK_WRITE); 369 else if (dir == KUAP_WRITE) 370 set_kuap(thread_amr | AMR_KUAP_BLOCK_READ); 371 else if (dir == KUAP_READ_WRITE) 372 set_kuap(thread_amr); 373 else 374 BUILD_BUG(); 375} 376 377#else /* CONFIG_PPC_KUAP */ 378 379static __always_inline unsigned long get_kuap(void) 380{ 381 return AMR_KUAP_BLOCKED; 382} 383 384static __always_inline void set_kuap(unsigned long value) { } 385 386static __always_inline void allow_user_access(void __user *to, const void __user *from, 387 unsigned long size, unsigned long dir) 388{ } 389 390#endif /* !CONFIG_PPC_KUAP */ 391 392static __always_inline void prevent_user_access(unsigned long dir) 393{ 394 set_kuap(AMR_KUAP_BLOCKED); 395 if (static_branch_unlikely(&uaccess_flush_key)) 396 do_uaccess_flush(); 397} 398 399static __always_inline unsigned long prevent_user_access_return(void) 400{ 401 unsigned long flags = get_kuap(); 402 403 set_kuap(AMR_KUAP_BLOCKED); 404 if (static_branch_unlikely(&uaccess_flush_key)) 405 do_uaccess_flush(); 406 407 return flags; 408} 409 410static __always_inline void restore_user_access(unsigned long flags) 411{ 412 set_kuap(flags); 413 if (static_branch_unlikely(&uaccess_flush_key) && flags == AMR_KUAP_BLOCKED) 414 do_uaccess_flush(); 415} 416#endif /* __ASSEMBLY__ */ 417 418#endif /* _ASM_POWERPC_BOOK3S_64_KUP_H */ 419