1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __VDSO_HELPERS_H
3#define __VDSO_HELPERS_H
4
5#ifndef __ASSEMBLY__
6
7#include <vdso/datapage.h>
8
9static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
10{
11	u32 seq;
12
13	while (unlikely((seq = READ_ONCE(vd->seq)) & 1))
14		cpu_relax();
15
16	smp_rmb();
17	return seq;
18}
19
20static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
21					   u32 start)
22{
23	u32 seq;
24
25	smp_rmb();
26	seq = READ_ONCE(vd->seq);
27	return seq != start;
28}
29
30static __always_inline void vdso_write_begin(struct vdso_data *vd)
31{
32	/*
33	 * WRITE_ONCE() is required otherwise the compiler can validly tear
34	 * updates to vd[x].seq and it is possible that the value seen by the
35	 * reader is inconsistent.
36	 */
37	WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
38	WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
39	smp_wmb();
40}
41
42static __always_inline void vdso_write_end(struct vdso_data *vd)
43{
44	smp_wmb();
45	/*
46	 * WRITE_ONCE() is required otherwise the compiler can validly tear
47	 * updates to vd[x].seq and it is possible that the value seen by the
48	 * reader is inconsistent.
49	 */
50	WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
51	WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
52}
53
54#endif /* !__ASSEMBLY__ */
55
56#endif /* __VDSO_HELPERS_H */
57