15780Sdg/* SPDX-License-Identifier: GPL-2.0 */ 25780Sdg#ifndef __ARCH_S390_PERCPU__ 35780Sdg#define __ARCH_S390_PERCPU__ 45780Sdg 55780Sdg#include <linux/preempt.h> 65780Sdg#include <asm/cmpxchg.h> 75780Sdg 85780Sdg/* 95780Sdg * s390 uses its own implementation for per cpu data, the offset of 105780Sdg * the cpu local data area is cached in the cpu's lowcore memory. 115780Sdg */ 125780Sdg#define __my_cpu_offset S390_lowcore.percpu_offset 135780Sdg 145780Sdg/* 155780Sdg * For 64 bit module code, the module may be more than 4G above the 165780Sdg * per cpu area, use weak definitions to force the compiler to 175780Sdg * generate external references. 185780Sdg */ 195780Sdg#if defined(MODULE) 205780Sdg#define ARCH_NEEDS_WEAK_PER_CPU 215780Sdg#endif 225780Sdg 235780Sdg/* 245780Sdg * We use a compare-and-swap loop since that uses less cpu cycles than 255780Sdg * disabling and enabling interrupts like the generic variant would do. 265780Sdg */ 275780Sdg#define arch_this_cpu_to_op_simple(pcp, val, op) \ 285780Sdg({ \ 295780Sdg typedef typeof(pcp) pcp_op_T__; \ 305780Sdg pcp_op_T__ old__, new__, prev__; \ 3185437Speter pcp_op_T__ *ptr__; \ 3293000Sobrien preempt_disable_notrace(); \ 335780Sdg ptr__ = raw_cpu_ptr(&(pcp)); \ 345780Sdg prev__ = READ_ONCE(*ptr__); \ 355780Sdg do { \ 365780Sdg old__ = prev__; \ 375780Sdg new__ = old__ op (val); \ 385780Sdg prev__ = cmpxchg(ptr__, old__, new__); \ 395780Sdg } while (prev__ != old__); \ 405780Sdg preempt_enable_notrace(); \ 41108533Sschweikh new__; \ 425780Sdg}) 435780Sdg 445780Sdg#define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 455780Sdg#define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 465780Sdg#define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 475780Sdg#define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 485780Sdg#define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 495780Sdg#define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 505780Sdg#define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 515780Sdg#define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 525780Sdg 535780Sdg#ifndef CONFIG_HAVE_MARCH_Z196_FEATURES 545780Sdg 555780Sdg#define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 565780Sdg#define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 575780Sdg#define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 585780Sdg#define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) 595780Sdg#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 605780Sdg#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &) 615780Sdg#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 625780Sdg#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) 635780Sdg 645780Sdg#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 655780Sdg 665780Sdg#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ 675780Sdg{ \ 685780Sdg typedef typeof(pcp) pcp_op_T__; \ 695780Sdg pcp_op_T__ val__ = (val); \ 705780Sdg pcp_op_T__ old__, *ptr__; \ 715780Sdg preempt_disable_notrace(); \ 725780Sdg ptr__ = raw_cpu_ptr(&(pcp)); \ 735780Sdg if (__builtin_constant_p(val__) && \ 745780Sdg ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ 755780Sdg asm volatile( \ 765780Sdg op2 " %[ptr__],%[val__]\n" \ 775780Sdg : [ptr__] "+Q" (*ptr__) \ 785780Sdg : [val__] "i" ((szcast)val__) \ 795780Sdg : "cc"); \ 805780Sdg } else { \ 815780Sdg asm volatile( \ 825780Sdg op1 " %[old__],%[val__],%[ptr__]\n" \ 835780Sdg : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 845780Sdg : [val__] "d" (val__) \ 855780Sdg : "cc"); \ 865780Sdg } \ 875780Sdg preempt_enable_notrace(); \ 885780Sdg} 895780Sdg 905780Sdg#define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) 915780Sdg#define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) 925780Sdg 935780Sdg#define arch_this_cpu_add_return(pcp, val, op) \ 945780Sdg({ \ 955780Sdg typedef typeof(pcp) pcp_op_T__; \ 965780Sdg pcp_op_T__ val__ = (val); \ 975780Sdg pcp_op_T__ old__, *ptr__; \ 985780Sdg preempt_disable_notrace(); \ 995780Sdg ptr__ = raw_cpu_ptr(&(pcp)); \ 100184548Speter asm volatile( \ 101217106Skib op " %[old__],%[val__],%[ptr__]\n" \ 102217106Skib : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 103 : [val__] "d" (val__) \ 104 : "cc"); \ 105 preempt_enable_notrace(); \ 106 old__ + val__; \ 107}) 108 109#define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa") 110#define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") 111 112#define arch_this_cpu_to_op(pcp, val, op) \ 113{ \ 114 typedef typeof(pcp) pcp_op_T__; \ 115 pcp_op_T__ val__ = (val); \ 116 pcp_op_T__ old__, *ptr__; \ 117 preempt_disable_notrace(); \ 118 ptr__ = raw_cpu_ptr(&(pcp)); \ 119 asm volatile( \ 120 op " %[old__],%[val__],%[ptr__]\n" \ 121 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ 122 : [val__] "d" (val__) \ 123 : "cc"); \ 124 preempt_enable_notrace(); \ 125} 126 127#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") 128#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") 129#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao") 130#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog") 131 132#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 133 134#define arch_this_cpu_cmpxchg(pcp, oval, nval) \ 135({ \ 136 typedef typeof(pcp) pcp_op_T__; \ 137 pcp_op_T__ ret__; \ 138 pcp_op_T__ *ptr__; \ 139 preempt_disable_notrace(); \ 140 ptr__ = raw_cpu_ptr(&(pcp)); \ 141 ret__ = cmpxchg(ptr__, oval, nval); \ 142 preempt_enable_notrace(); \ 143 ret__; \ 144}) 145 146#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 147#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 148#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 149#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) 150 151#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) 152 153#define this_cpu_cmpxchg128(pcp, oval, nval) \ 154({ \ 155 typedef typeof(pcp) pcp_op_T__; \ 156 u128 old__, new__, ret__; \ 157 pcp_op_T__ *ptr__; \ 158 old__ = oval; \ 159 new__ = nval; \ 160 preempt_disable_notrace(); \ 161 ptr__ = raw_cpu_ptr(&(pcp)); \ 162 ret__ = cmpxchg128((void *)ptr__, old__, new__); \ 163 preempt_enable_notrace(); \ 164 ret__; \ 165}) 166 167#define arch_this_cpu_xchg(pcp, nval) \ 168({ \ 169 typeof(pcp) *ptr__; \ 170 typeof(pcp) ret__; \ 171 preempt_disable_notrace(); \ 172 ptr__ = raw_cpu_ptr(&(pcp)); \ 173 ret__ = xchg(ptr__, nval); \ 174 preempt_enable_notrace(); \ 175 ret__; \ 176}) 177 178#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval) 179#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval) 180#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval) 181#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval) 182 183#include <asm-generic/percpu.h> 184 185#endif /* __ARCH_S390_PERCPU__ */ 186