1/*
2 * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#ifndef OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP
26#define OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP
27
28#include "runtime/os.hpp"
29#include "vm_version_arm.hpp"
30
31// Implementation of class atomic
32
33/*
34 * Atomic long operations on 32-bit ARM
35 * ARM v7 supports LDREXD/STREXD synchronization instructions so no problem.
36 * ARM < v7 does not have explicit 64 atomic load/store capability.
37 * However, gcc emits LDRD/STRD instructions on v5te and LDM/STM on v5t
38 * when loading/storing 64 bits.
39 * For non-MP machines (which is all we support for ARM < v7)
40 * under current Linux distros these instructions appear atomic.
41 * See section A3.5.3 of ARM Architecture Reference Manual for ARM v7.
42 * Also, for cmpxchg64, if ARM < v7 we check for cmpxchg64 support in the
43 * Linux kernel using _kuser_helper_version. See entry-armv.S in the Linux
44 * kernel source or kernel_user_helpers.txt in Linux Doc.
45 */
46
47inline void Atomic::store    (jbyte    store_value, jbyte*    dest) { *dest = store_value; }
48inline void Atomic::store    (jshort   store_value, jshort*   dest) { *dest = store_value; }
49inline void Atomic::store    (jint     store_value, jint*     dest) { *dest = store_value; }
50inline void Atomic::store_ptr(intptr_t store_value, intptr_t* dest) { *dest = store_value; }
51inline void Atomic::store_ptr(void*    store_value, void*     dest) { *(void**)dest = store_value; }
52
53inline void Atomic::store    (jbyte    store_value, volatile jbyte*    dest) { *dest = store_value; }
54inline void Atomic::store    (jshort   store_value, volatile jshort*   dest) { *dest = store_value; }
55inline void Atomic::store    (jint     store_value, volatile jint*     dest) { *dest = store_value; }
56inline void Atomic::store_ptr(intptr_t store_value, volatile intptr_t* dest) { *dest = store_value; }
57inline void Atomic::store_ptr(void*    store_value, volatile void*     dest) { *(void* volatile *)dest = store_value; }
58
59inline jlong Atomic::load (const volatile jlong* src) {
60  assert(((intx)src & (sizeof(jlong)-1)) == 0, "Atomic load jlong mis-aligned");
61#ifdef AARCH64
62  return *src;
63#else
64  return (*os::atomic_load_long_func)(src);
65#endif
66}
67
68inline void Atomic::store (jlong value, volatile jlong* dest) {
69  assert(((intx)dest & (sizeof(jlong)-1)) == 0, "Atomic store jlong mis-aligned");
70#ifdef AARCH64
71  *dest = value;
72#else
73  (*os::atomic_store_long_func)(value, dest);
74#endif
75}
76
77inline void Atomic::store (jlong value, jlong* dest) {
78  store(value, (volatile jlong*)dest);
79}
80
81// As per atomic.hpp all read-modify-write operations have to provide two-way
82// barriers semantics. For AARCH64 we are using load-acquire-with-reservation and
83// store-release-with-reservation. While load-acquire combined with store-release
84// do not generally form two-way barriers, their use with reservations does - the
85// ARMv8 architecture manual Section F "Barrier Litmus Tests" indicates they
86// provide sequentially consistent semantics. All we need to add is an explicit
87// barrier in the failure path of the cmpxchg operations (as these don't execute
88// the store) - arguably this may be overly cautious as there is a very low
89// likelihood that the hardware would pull loads/stores into the region guarded
90// by the reservation.
91//
92// For ARMv7 we add explicit barriers in the stubs.
93
94template<size_t byte_size>
95struct Atomic::PlatformAdd
96  : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> >
97{
98  template<typename I, typename D>
99  D add_and_fetch(I add_value, D volatile* dest) const;
100};
101
102template<>
103template<typename I, typename D>
104inline D Atomic::PlatformAdd<4>::add_and_fetch(I add_value, D volatile* dest) const {
105  STATIC_ASSERT(4 == sizeof(I));
106  STATIC_ASSERT(4 == sizeof(D));
107#ifdef AARCH64
108  D val;
109  int tmp;
110  __asm__ volatile(
111    "1:\n\t"
112    " ldaxr %w[val], [%[dest]]\n\t"
113    " add %w[val], %w[val], %w[add_val]\n\t"
114    " stlxr %w[tmp], %w[val], [%[dest]]\n\t"
115    " cbnz %w[tmp], 1b\n\t"
116    : [val] "=&r" (val), [tmp] "=&r" (tmp)
117    : [add_val] "r" (add_value), [dest] "r" (dest)
118    : "memory");
119  return val;
120#else
121  return add_using_helper<jint>(os::atomic_add_func, add_value, dest);
122#endif
123}
124
125inline void Atomic::inc(volatile jint* dest) {
126  Atomic::add(1, (volatile jint *)dest);
127}
128
129inline void Atomic::dec(volatile jint* dest) {
130  Atomic::add(-1, (volatile jint *)dest);
131}
132
133#ifdef AARCH64
134template<>
135template<typename I, typename D>
136inline D Atomic::PlatformAdd<8>::add_and_fetch(I add_value, D volatile* dest) const {
137  STATIC_ASSERT(8 == sizeof(I));
138  STATIC_ASSERT(8 == sizeof(D));
139  D val;
140  int tmp;
141  __asm__ volatile(
142    "1:\n\t"
143    " ldaxr %[val], [%[dest]]\n\t"
144    " add %[val], %[val], %[add_val]\n\t"
145    " stlxr %w[tmp], %[val], [%[dest]]\n\t"
146    " cbnz %w[tmp], 1b\n\t"
147    : [val] "=&r" (val), [tmp] "=&r" (tmp)
148    : [add_val] "r" (add_value), [dest] "r" (dest)
149    : "memory");
150  return val;
151}
152#endif // AARCH64
153
154inline void Atomic::inc_ptr(volatile intptr_t* dest) {
155  Atomic::add_ptr(1, dest);
156}
157
158inline void Atomic::dec_ptr(volatile intptr_t* dest) {
159  Atomic::add_ptr(-1, dest);
160}
161
162inline void Atomic::inc_ptr(volatile void* dest) {
163  inc_ptr((volatile intptr_t*)dest);
164}
165
166inline void Atomic::dec_ptr(volatile void* dest) {
167  dec_ptr((volatile intptr_t*)dest);
168}
169
170
171inline jint Atomic::xchg(jint exchange_value, volatile jint* dest) {
172#ifdef AARCH64
173  jint old_val;
174  int tmp;
175  __asm__ volatile(
176    "1:\n\t"
177    " ldaxr %w[old_val], [%[dest]]\n\t"
178    " stlxr %w[tmp], %w[new_val], [%[dest]]\n\t"
179    " cbnz %w[tmp], 1b\n\t"
180    : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp)
181    : [new_val] "r" (exchange_value), [dest] "r" (dest)
182    : "memory");
183  return old_val;
184#else
185  return (*os::atomic_xchg_func)(exchange_value, dest);
186#endif
187}
188
189inline intptr_t Atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest) {
190#ifdef AARCH64
191  intptr_t old_val;
192  int tmp;
193  __asm__ volatile(
194    "1:\n\t"
195    " ldaxr %[old_val], [%[dest]]\n\t"
196    " stlxr %w[tmp], %[new_val], [%[dest]]\n\t"
197    " cbnz %w[tmp], 1b\n\t"
198    : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp)
199    : [new_val] "r" (exchange_value), [dest] "r" (dest)
200    : "memory");
201  return old_val;
202#else
203  return (intptr_t)xchg((jint)exchange_value, (volatile jint*)dest);
204#endif
205}
206
207inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
208  return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
209}
210
211// The memory_order parameter is ignored - we always provide the strongest/most-conservative ordering
212
213// No direct support for cmpxchg of bytes; emulate using int.
214template<>
215struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {};
216
217#ifndef AARCH64
218
219inline jint reorder_cmpxchg_func(jint exchange_value,
220                                 jint volatile* dest,
221                                 jint compare_value) {
222  // Warning:  Arguments are swapped to avoid moving them for kernel call
223  return (*os::atomic_cmpxchg_func)(compare_value, exchange_value, dest);
224}
225
226inline jlong reorder_cmpxchg_long_func(jlong exchange_value,
227                                       jlong volatile* dest,
228                                       jlong compare_value) {
229  assert(VM_Version::supports_cx8(), "Atomic compare and exchange jlong not supported on this architecture!");
230  // Warning:  Arguments are swapped to avoid moving them for kernel call
231  return (*os::atomic_cmpxchg_long_func)(compare_value, exchange_value, dest);
232}
233
234#endif // !AARCH64
235
236template<>
237template<typename T>
238inline T Atomic::PlatformCmpxchg<4>::operator()(T exchange_value,
239                                                T volatile* dest,
240                                                T compare_value,
241                                                cmpxchg_memory_order order) const {
242  STATIC_ASSERT(4 == sizeof(T));
243#ifdef AARCH64
244  T rv;
245  int tmp;
246  __asm__ volatile(
247    "1:\n\t"
248    " ldaxr %w[rv], [%[dest]]\n\t"
249    " cmp %w[rv], %w[cv]\n\t"
250    " b.ne 2f\n\t"
251    " stlxr %w[tmp], %w[ev], [%[dest]]\n\t"
252    " cbnz %w[tmp], 1b\n\t"
253    " b 3f\n\t"
254    "2:\n\t"
255    " dmb sy\n\t"
256    "3:\n\t"
257    : [rv] "=&r" (rv), [tmp] "=&r" (tmp)
258    : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value)
259    : "memory");
260  return rv;
261#else
262  return cmpxchg_using_helper<jint>(reorder_cmpxchg_func, exchange_value, dest, compare_value);
263#endif
264}
265
266template<>
267template<typename T>
268inline T Atomic::PlatformCmpxchg<8>::operator()(T exchange_value,
269                                                T volatile* dest,
270                                                T compare_value,
271                                                cmpxchg_memory_order order) const {
272  STATIC_ASSERT(8 == sizeof(T));
273#ifdef AARCH64
274  T rv;
275  int tmp;
276  __asm__ volatile(
277    "1:\n\t"
278    " ldaxr %[rv], [%[dest]]\n\t"
279    " cmp %[rv], %[cv]\n\t"
280    " b.ne 2f\n\t"
281    " stlxr %w[tmp], %[ev], [%[dest]]\n\t"
282    " cbnz %w[tmp], 1b\n\t"
283    " b 3f\n\t"
284    "2:\n\t"
285    " dmb sy\n\t"
286    "3:\n\t"
287    : [rv] "=&r" (rv), [tmp] "=&r" (tmp)
288    : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value)
289    : "memory");
290  return rv;
291#else
292  return cmpxchg_using_helper<jlong>(reorder_cmpxchg_long_func, exchange_value, dest, compare_value);
293#endif
294}
295
296#endif // OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP
297