1/*
2 * Copyright 2014 Jaidev Sridhar.
3 * Copyright 2014 Samy Al Bahra.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#ifndef CK_SWLOCK_H
29#define CK_SWLOCK_H
30
31#include <ck_elide.h>
32#include <ck_limits.h>
33#include <ck_pr.h>
34#include <ck_stdbool.h>
35#include <ck_stddef.h>
36
37struct ck_swlock {
38	uint32_t value;
39};
40typedef struct ck_swlock ck_swlock_t;
41
42#define CK_SWLOCK_INITIALIZER	{0}
43#define CK_SWLOCK_WRITER_BIT	(1UL << 31)
44#define CK_SWLOCK_LATCH_BIT	(1UL << 30)
45#define CK_SWLOCK_WRITER_MASK	(CK_SWLOCK_LATCH_BIT | CK_SWLOCK_WRITER_BIT)
46#define CK_SWLOCK_READER_MASK   (UINT32_MAX ^ CK_SWLOCK_WRITER_MASK)
47
48CK_CC_INLINE static void
49ck_swlock_init(struct ck_swlock *rw)
50{
51
52	rw->value = 0;
53	ck_pr_barrier();
54	return;
55}
56
57CK_CC_INLINE static void
58ck_swlock_write_unlock(ck_swlock_t *rw)
59{
60
61	ck_pr_fence_unlock();
62	ck_pr_and_32(&rw->value, CK_SWLOCK_READER_MASK);
63	return;
64}
65
66CK_CC_INLINE static bool
67ck_swlock_locked_writer(ck_swlock_t *rw)
68{
69	bool r;
70
71	r = ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT;
72	ck_pr_fence_acquire();
73	return r;
74}
75
76CK_CC_INLINE static void
77ck_swlock_write_downgrade(ck_swlock_t *rw)
78{
79
80	ck_pr_inc_32(&rw->value);
81	ck_swlock_write_unlock(rw);
82	return;
83}
84
85CK_CC_INLINE static bool
86ck_swlock_locked(ck_swlock_t *rw)
87{
88	bool r;
89
90	r = ck_pr_load_32(&rw->value);
91	ck_pr_fence_acquire();
92	return r;
93}
94
95CK_CC_INLINE static bool
96ck_swlock_write_trylock(ck_swlock_t *rw)
97{
98	bool r;
99
100	r = ck_pr_cas_32(&rw->value, 0, CK_SWLOCK_WRITER_BIT);
101	ck_pr_fence_lock();
102	return r;
103}
104
105CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_write, ck_swlock_t,
106    ck_swlock_locked, ck_swlock_write_trylock)
107
108CK_CC_INLINE static void
109ck_swlock_write_lock(ck_swlock_t *rw)
110{
111
112	ck_pr_or_32(&rw->value, CK_SWLOCK_WRITER_BIT);
113	while (ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_MASK)
114		ck_pr_stall();
115
116	ck_pr_fence_lock();
117	return;
118}
119
120CK_CC_INLINE static void
121ck_swlock_write_latch(ck_swlock_t *rw)
122{
123
124	/* Publish intent to acquire lock. */
125	ck_pr_or_32(&rw->value, CK_SWLOCK_WRITER_BIT);
126
127	/* Stall until readers have seen the writer and cleared. */
128	while (ck_pr_cas_32(&rw->value, CK_SWLOCK_WRITER_BIT,
129	    CK_SWLOCK_WRITER_MASK) == false)  {
130		do {
131			ck_pr_stall();
132		} while (ck_pr_load_32(&rw->value) != CK_SWLOCK_WRITER_BIT);
133	}
134
135	ck_pr_fence_lock();
136	return;
137}
138
139CK_CC_INLINE static void
140ck_swlock_write_unlatch(ck_swlock_t *rw)
141{
142
143	ck_pr_fence_unlock();
144	ck_pr_store_32(&rw->value, 0);
145	return;
146}
147
148CK_ELIDE_PROTOTYPE(ck_swlock_write, ck_swlock_t,
149    ck_swlock_locked, ck_swlock_write_lock,
150    ck_swlock_locked_writer, ck_swlock_write_unlock)
151
152CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_read, ck_swlock_t,
153    ck_swlock_locked_writer, ck_swlock_read_trylock)
154
155CK_CC_INLINE static bool
156ck_swlock_read_trylock(ck_swlock_t *rw)
157{
158	uint32_t l = ck_pr_load_32(&rw->value);
159
160	if (l & CK_SWLOCK_WRITER_BIT)
161		return false;
162
163	l = ck_pr_faa_32(&rw->value, 1) & CK_SWLOCK_WRITER_MASK;
164	if (l == CK_SWLOCK_WRITER_BIT)
165		ck_pr_dec_32(&rw->value);
166
167	ck_pr_fence_lock();
168	return l == 0;
169}
170
171CK_CC_INLINE static void
172ck_swlock_read_lock(ck_swlock_t *rw)
173{
174	uint32_t l;
175
176	for (;;) {
177		while (ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT)
178			ck_pr_stall();
179
180		l = ck_pr_faa_32(&rw->value, 1) & CK_SWLOCK_WRITER_MASK;
181		if (l == 0)
182			break;
183
184		/*
185		 * If the latch bit has not been set, then the writer would
186		 * have observed the reader and will wait to completion of
187		 * read-side critical section.
188		 */
189		if (l == CK_SWLOCK_WRITER_BIT)
190			ck_pr_dec_32(&rw->value);
191	}
192
193	ck_pr_fence_lock();
194	return;
195}
196
197CK_CC_INLINE static bool
198ck_swlock_locked_reader(ck_swlock_t *rw)
199{
200
201	ck_pr_fence_load();
202	return ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_MASK;
203}
204
205CK_CC_INLINE static void
206ck_swlock_read_unlock(ck_swlock_t *rw)
207{
208
209	ck_pr_fence_unlock();
210	ck_pr_dec_32(&rw->value);
211	return;
212}
213
214CK_ELIDE_PROTOTYPE(ck_swlock_read, ck_swlock_t,
215    ck_swlock_locked_writer, ck_swlock_read_lock,
216    ck_swlock_locked_reader, ck_swlock_read_unlock)
217
218#endif /* CK_SWLOCK_H */
219