1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
24 */
25
26#include <sys/zfs_context.h>
27#include <sys/refcount.h>
28
29#ifdef	ZFS_DEBUG
30
31#ifdef _KERNEL
32int reference_tracking_enable = FALSE; /* runs out of memory too easily */
33SYSCTL_DECL(_vfs_zfs);
34SYSCTL_INT(_vfs_zfs, OID_AUTO, reference_tracking_enable, CTLFLAG_RDTUN,
35    &reference_tracking_enable, 0,
36    "Track reference holders to refcount_t objects, used mostly by ZFS");
37#else
38int reference_tracking_enable = TRUE;
39#endif
40int reference_history = 3; /* tunable */
41
42static kmem_cache_t *reference_cache;
43static kmem_cache_t *reference_history_cache;
44
45void
46refcount_sysinit(void)
47{
48	reference_cache = kmem_cache_create("reference_cache",
49	    sizeof (reference_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
50
51	reference_history_cache = kmem_cache_create("reference_history_cache",
52	    sizeof (uint64_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
53}
54
55void
56refcount_fini(void)
57{
58	kmem_cache_destroy(reference_cache);
59	kmem_cache_destroy(reference_history_cache);
60}
61
62void
63refcount_create(refcount_t *rc)
64{
65	mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL);
66	list_create(&rc->rc_list, sizeof (reference_t),
67	    offsetof(reference_t, ref_link));
68	list_create(&rc->rc_removed, sizeof (reference_t),
69	    offsetof(reference_t, ref_link));
70	rc->rc_count = 0;
71	rc->rc_removed_count = 0;
72	rc->rc_tracked = reference_tracking_enable;
73}
74
75void
76refcount_create_tracked(refcount_t *rc)
77{
78	refcount_create(rc);
79	rc->rc_tracked = B_TRUE;
80}
81
82void
83refcount_create_untracked(refcount_t *rc)
84{
85	refcount_create(rc);
86	rc->rc_tracked = B_FALSE;
87}
88
89void
90refcount_destroy_many(refcount_t *rc, uint64_t number)
91{
92	reference_t *ref;
93
94	ASSERT(rc->rc_count == number);
95	while (ref = list_head(&rc->rc_list)) {
96		list_remove(&rc->rc_list, ref);
97		kmem_cache_free(reference_cache, ref);
98	}
99	list_destroy(&rc->rc_list);
100
101	while (ref = list_head(&rc->rc_removed)) {
102		list_remove(&rc->rc_removed, ref);
103		kmem_cache_free(reference_history_cache, ref->ref_removed);
104		kmem_cache_free(reference_cache, ref);
105	}
106	list_destroy(&rc->rc_removed);
107	mutex_destroy(&rc->rc_mtx);
108}
109
110void
111refcount_destroy(refcount_t *rc)
112{
113	refcount_destroy_many(rc, 0);
114}
115
116int
117refcount_is_zero(refcount_t *rc)
118{
119	return (rc->rc_count == 0);
120}
121
122int64_t
123refcount_count(refcount_t *rc)
124{
125	return (rc->rc_count);
126}
127
128int64_t
129refcount_add_many(refcount_t *rc, uint64_t number, void *holder)
130{
131	reference_t *ref = NULL;
132	int64_t count;
133
134	if (rc->rc_tracked) {
135		ref = kmem_cache_alloc(reference_cache, KM_SLEEP);
136		ref->ref_holder = holder;
137		ref->ref_number = number;
138	}
139	mutex_enter(&rc->rc_mtx);
140	ASSERT(rc->rc_count >= 0);
141	if (rc->rc_tracked)
142		list_insert_head(&rc->rc_list, ref);
143	rc->rc_count += number;
144	count = rc->rc_count;
145	mutex_exit(&rc->rc_mtx);
146
147	return (count);
148}
149
150int64_t
151refcount_add(refcount_t *rc, void *holder)
152{
153	return (refcount_add_many(rc, 1, holder));
154}
155
156int64_t
157refcount_remove_many(refcount_t *rc, uint64_t number, void *holder)
158{
159	reference_t *ref;
160	int64_t count;
161
162	mutex_enter(&rc->rc_mtx);
163	ASSERT(rc->rc_count >= number);
164
165	if (!rc->rc_tracked) {
166		rc->rc_count -= number;
167		count = rc->rc_count;
168		mutex_exit(&rc->rc_mtx);
169		return (count);
170	}
171
172	for (ref = list_head(&rc->rc_list); ref;
173	    ref = list_next(&rc->rc_list, ref)) {
174		if (ref->ref_holder == holder && ref->ref_number == number) {
175			list_remove(&rc->rc_list, ref);
176			if (reference_history > 0) {
177				ref->ref_removed =
178				    kmem_cache_alloc(reference_history_cache,
179				    KM_SLEEP);
180				list_insert_head(&rc->rc_removed, ref);
181				rc->rc_removed_count++;
182				if (rc->rc_removed_count > reference_history) {
183					ref = list_tail(&rc->rc_removed);
184					list_remove(&rc->rc_removed, ref);
185					kmem_cache_free(reference_history_cache,
186					    ref->ref_removed);
187					kmem_cache_free(reference_cache, ref);
188					rc->rc_removed_count--;
189				}
190			} else {
191				kmem_cache_free(reference_cache, ref);
192			}
193			rc->rc_count -= number;
194			count = rc->rc_count;
195			mutex_exit(&rc->rc_mtx);
196			return (count);
197		}
198	}
199	panic("No such hold %p on refcount %llx", holder,
200	    (u_longlong_t)(uintptr_t)rc);
201	return (-1);
202}
203
204int64_t
205refcount_remove(refcount_t *rc, void *holder)
206{
207	return (refcount_remove_many(rc, 1, holder));
208}
209
210void
211refcount_transfer(refcount_t *dst, refcount_t *src)
212{
213	int64_t count, removed_count;
214	list_t list, removed;
215
216	list_create(&list, sizeof (reference_t),
217	    offsetof(reference_t, ref_link));
218	list_create(&removed, sizeof (reference_t),
219	    offsetof(reference_t, ref_link));
220
221	mutex_enter(&src->rc_mtx);
222	count = src->rc_count;
223	removed_count = src->rc_removed_count;
224	src->rc_count = 0;
225	src->rc_removed_count = 0;
226	list_move_tail(&list, &src->rc_list);
227	list_move_tail(&removed, &src->rc_removed);
228	mutex_exit(&src->rc_mtx);
229
230	mutex_enter(&dst->rc_mtx);
231	dst->rc_count += count;
232	dst->rc_removed_count += removed_count;
233	list_move_tail(&dst->rc_list, &list);
234	list_move_tail(&dst->rc_removed, &removed);
235	mutex_exit(&dst->rc_mtx);
236
237	list_destroy(&list);
238	list_destroy(&removed);
239}
240
241void
242refcount_transfer_ownership(refcount_t *rc, void *current_holder,
243    void *new_holder)
244{
245	reference_t *ref;
246	boolean_t found = B_FALSE;
247
248	mutex_enter(&rc->rc_mtx);
249	if (!rc->rc_tracked) {
250		mutex_exit(&rc->rc_mtx);
251		return;
252	}
253
254	for (ref = list_head(&rc->rc_list); ref;
255	    ref = list_next(&rc->rc_list, ref)) {
256		if (ref->ref_holder == current_holder) {
257			ref->ref_holder = new_holder;
258			found = B_TRUE;
259			break;
260		}
261	}
262	ASSERT(found);
263	mutex_exit(&rc->rc_mtx);
264}
265
266/*
267 * If tracking is enabled, return true if a reference exists that matches
268 * the "holder" tag. If tracking is disabled, then return true if a reference
269 * might be held.
270 */
271boolean_t
272refcount_held(refcount_t *rc, void *holder)
273{
274	reference_t *ref;
275
276	mutex_enter(&rc->rc_mtx);
277
278	if (!rc->rc_tracked) {
279		mutex_exit(&rc->rc_mtx);
280		return (rc->rc_count > 0);
281	}
282
283	for (ref = list_head(&rc->rc_list); ref;
284	    ref = list_next(&rc->rc_list, ref)) {
285		if (ref->ref_holder == holder) {
286			mutex_exit(&rc->rc_mtx);
287			return (B_TRUE);
288		}
289	}
290	mutex_exit(&rc->rc_mtx);
291	return (B_FALSE);
292}
293
294/*
295 * If tracking is enabled, return true if a reference does not exist that
296 * matches the "holder" tag. If tracking is disabled, always return true
297 * since the reference might not be held.
298 */
299boolean_t
300refcount_not_held(refcount_t *rc, void *holder)
301{
302	reference_t *ref;
303
304	mutex_enter(&rc->rc_mtx);
305
306	if (!rc->rc_tracked) {
307		mutex_exit(&rc->rc_mtx);
308		return (B_TRUE);
309	}
310
311	for (ref = list_head(&rc->rc_list); ref;
312	    ref = list_next(&rc->rc_list, ref)) {
313		if (ref->ref_holder == holder) {
314			mutex_exit(&rc->rc_mtx);
315			return (B_FALSE);
316		}
317	}
318	mutex_exit(&rc->rc_mtx);
319	return (B_TRUE);
320}
321#endif	/* ZFS_DEBUG */
322