1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Based on arch/arm/include/asm/barrier.h 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 */ 7#ifndef __ASM_BARRIER_H 8#define __ASM_BARRIER_H 9 10#ifndef __ASSEMBLY__ 11 12#include <linux/kasan-checks.h> 13 14#include <asm/alternative-macros.h> 15 16#define __nops(n) ".rept " #n "\nnop\n.endr\n" 17#define nops(n) asm volatile(__nops(n)) 18 19#define sev() asm volatile("sev" : : : "memory") 20#define wfe() asm volatile("wfe" : : : "memory") 21#define wfet(val) asm volatile("msr s0_3_c1_c0_0, %0" \ 22 : : "r" (val) : "memory") 23#define wfi() asm volatile("wfi" : : : "memory") 24#define wfit(val) asm volatile("msr s0_3_c1_c0_1, %0" \ 25 : : "r" (val) : "memory") 26 27#define isb() asm volatile("isb" : : : "memory") 28#define dmb(opt) asm volatile("dmb " #opt : : : "memory") 29#define dsb(opt) asm volatile("dsb " #opt : : : "memory") 30 31#define psb_csync() asm volatile("hint #17" : : : "memory") 32#define __tsb_csync() asm volatile("hint #18" : : : "memory") 33#define csdb() asm volatile("hint #20" : : : "memory") 34 35/* 36 * Data Gathering Hint: 37 * This instruction prevents merging memory accesses with Normal-NC or 38 * Device-GRE attributes before the hint instruction with any memory accesses 39 * appearing after the hint instruction. 40 */ 41#define dgh() asm volatile("hint #6" : : : "memory") 42 43#ifdef CONFIG_ARM64_PSEUDO_NMI 44#define pmr_sync() \ 45 do { \ 46 asm volatile( \ 47 ALTERNATIVE_CB("dsb sy", \ 48 ARM64_HAS_GIC_PRIO_RELAXED_SYNC, \ 49 alt_cb_patch_nops) \ 50 ); \ 51 } while(0) 52#else 53#define pmr_sync() do {} while (0) 54#endif 55 56#define __mb() dsb(sy) 57#define __rmb() dsb(ld) 58#define __wmb() dsb(st) 59 60#define __dma_mb() dmb(osh) 61#define __dma_rmb() dmb(oshld) 62#define __dma_wmb() dmb(oshst) 63 64#define io_stop_wc() dgh() 65 66#define tsb_csync() \ 67 do { \ 68 /* \ 69 * CPUs affected by Arm Erratum 2054223 or 2067961 needs \ 70 * another TSB to ensure the trace is flushed. The barriers \ 71 * don't have to be strictly back to back, as long as the \ 72 * CPU is in trace prohibited state. \ 73 */ \ 74 if (cpus_have_final_cap(ARM64_WORKAROUND_TSB_FLUSH_FAILURE)) \ 75 __tsb_csync(); \ 76 __tsb_csync(); \ 77 } while (0) 78 79/* 80 * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz 81 * and 0 otherwise. 82 */ 83#define array_index_mask_nospec array_index_mask_nospec 84static inline unsigned long array_index_mask_nospec(unsigned long idx, 85 unsigned long sz) 86{ 87 unsigned long mask; 88 89 asm volatile( 90 " cmp %1, %2\n" 91 " sbc %0, xzr, xzr\n" 92 : "=r" (mask) 93 : "r" (idx), "Ir" (sz) 94 : "cc"); 95 96 csdb(); 97 return mask; 98} 99 100/* 101 * Ensure that reads of the counter are treated the same as memory reads 102 * for the purposes of ordering by subsequent memory barriers. 103 * 104 * This insanity brought to you by speculative system register reads, 105 * out-of-order memory accesses, sequence locks and Thomas Gleixner. 106 * 107 * https://lore.kernel.org/r/alpine.DEB.2.21.1902081950260.1662@nanos.tec.linutronix.de/ 108 */ 109#define arch_counter_enforce_ordering(val) do { \ 110 u64 tmp, _val = (val); \ 111 \ 112 asm volatile( \ 113 " eor %0, %1, %1\n" \ 114 " add %0, sp, %0\n" \ 115 " ldr xzr, [%0]" \ 116 : "=r" (tmp) : "r" (_val)); \ 117} while (0) 118 119#define __smp_mb() dmb(ish) 120#define __smp_rmb() dmb(ishld) 121#define __smp_wmb() dmb(ishst) 122 123#define __smp_store_release(p, v) \ 124do { \ 125 typeof(p) __p = (p); \ 126 union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u = \ 127 { .__val = (__force __unqual_scalar_typeof(*p)) (v) }; \ 128 compiletime_assert_atomic_type(*p); \ 129 kasan_check_write(__p, sizeof(*p)); \ 130 switch (sizeof(*p)) { \ 131 case 1: \ 132 asm volatile ("stlrb %w1, %0" \ 133 : "=Q" (*__p) \ 134 : "rZ" (*(__u8 *)__u.__c) \ 135 : "memory"); \ 136 break; \ 137 case 2: \ 138 asm volatile ("stlrh %w1, %0" \ 139 : "=Q" (*__p) \ 140 : "rZ" (*(__u16 *)__u.__c) \ 141 : "memory"); \ 142 break; \ 143 case 4: \ 144 asm volatile ("stlr %w1, %0" \ 145 : "=Q" (*__p) \ 146 : "rZ" (*(__u32 *)__u.__c) \ 147 : "memory"); \ 148 break; \ 149 case 8: \ 150 asm volatile ("stlr %x1, %0" \ 151 : "=Q" (*__p) \ 152 : "rZ" (*(__u64 *)__u.__c) \ 153 : "memory"); \ 154 break; \ 155 } \ 156} while (0) 157 158#define __smp_load_acquire(p) \ 159({ \ 160 union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u; \ 161 typeof(p) __p = (p); \ 162 compiletime_assert_atomic_type(*p); \ 163 kasan_check_read(__p, sizeof(*p)); \ 164 switch (sizeof(*p)) { \ 165 case 1: \ 166 asm volatile ("ldarb %w0, %1" \ 167 : "=r" (*(__u8 *)__u.__c) \ 168 : "Q" (*__p) : "memory"); \ 169 break; \ 170 case 2: \ 171 asm volatile ("ldarh %w0, %1" \ 172 : "=r" (*(__u16 *)__u.__c) \ 173 : "Q" (*__p) : "memory"); \ 174 break; \ 175 case 4: \ 176 asm volatile ("ldar %w0, %1" \ 177 : "=r" (*(__u32 *)__u.__c) \ 178 : "Q" (*__p) : "memory"); \ 179 break; \ 180 case 8: \ 181 asm volatile ("ldar %0, %1" \ 182 : "=r" (*(__u64 *)__u.__c) \ 183 : "Q" (*__p) : "memory"); \ 184 break; \ 185 } \ 186 (typeof(*p))__u.__val; \ 187}) 188 189#define smp_cond_load_relaxed(ptr, cond_expr) \ 190({ \ 191 typeof(ptr) __PTR = (ptr); \ 192 __unqual_scalar_typeof(*ptr) VAL; \ 193 for (;;) { \ 194 VAL = READ_ONCE(*__PTR); \ 195 if (cond_expr) \ 196 break; \ 197 __cmpwait_relaxed(__PTR, VAL); \ 198 } \ 199 (typeof(*ptr))VAL; \ 200}) 201 202#define smp_cond_load_acquire(ptr, cond_expr) \ 203({ \ 204 typeof(ptr) __PTR = (ptr); \ 205 __unqual_scalar_typeof(*ptr) VAL; \ 206 for (;;) { \ 207 VAL = smp_load_acquire(__PTR); \ 208 if (cond_expr) \ 209 break; \ 210 __cmpwait_relaxed(__PTR, VAL); \ 211 } \ 212 (typeof(*ptr))VAL; \ 213}) 214 215#include <asm-generic/barrier.h> 216 217#endif /* __ASSEMBLY__ */ 218 219#endif /* __ASM_BARRIER_H */ 220