subr_pserialize.c revision 1.16 
1/*	$NetBSD: subr_pserialize.c,v 1.16 2019/12/05 03:21:17 riastradh Exp $	*/
2
3/*-
4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/*
30 * Passive serialization.
31 */
32
33#include <sys/cdefs.h>
34__KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.16 2019/12/05 03:21:17 riastradh Exp $");
35
36#include <sys/param.h>
37#include <sys/atomic.h>
38#include <sys/cpu.h>
39#include <sys/evcnt.h>
40#include <sys/kmem.h>
41#include <sys/mutex.h>
42#include <sys/pserialize.h>
43#include <sys/xcall.h>
44
45struct pserialize {
46	lwp_t *			psz_owner;
47};
48
49static kmutex_t			psz_lock	__cacheline_aligned;
50static struct evcnt		psz_ev_excl	__cacheline_aligned;
51
52/*
53 * pserialize_init:
54 *
55 *	Initialize passive serialization structures.
56 */
57void
58pserialize_init(void)
59{
60
61	mutex_init(&psz_lock, MUTEX_DEFAULT, IPL_NONE);
62	evcnt_attach_dynamic(&psz_ev_excl, EVCNT_TYPE_MISC, NULL,
63	    "pserialize", "exclusive access");
64}
65
66/*
67 * pserialize_create:
68 *
69 *	Create and initialize a passive serialization object.
70 */
71pserialize_t
72pserialize_create(void)
73{
74	pserialize_t psz;
75
76	psz = kmem_zalloc(sizeof(*psz), KM_SLEEP);
77	return psz;
78}
79
80/*
81 * pserialize_destroy:
82 *
83 *	Destroy a passive serialization object.
84 */
85void
86pserialize_destroy(pserialize_t psz)
87{
88
89	KASSERT(psz->psz_owner == NULL);
90	kmem_free(psz, sizeof(*psz));
91}
92
93/*
94 * pserialize_perform:
95 *
96 *	Perform the write side of passive serialization.  This operation
97 *	MUST be serialized at a caller level (e.g. with a mutex or by a
98 *	single-threaded use).
99 */
100void
101pserialize_perform(pserialize_t psz)
102{
103
104	KASSERT(!cpu_intr_p());
105	KASSERT(!cpu_softintr_p());
106
107	if (__predict_false(panicstr != NULL)) {
108		return;
109	}
110	KASSERT(psz->psz_owner == NULL);
111
112	if (__predict_false(mp_online == false)) {
113		psz_ev_excl.ev_count++;
114		return;
115	}
116
117	psz->psz_owner = curlwp;
118
119	/*
120	 * Broadcast a NOP to all CPUs and wait until all of them complete.
121	 */
122	xc_barrier(XC_HIGHPRI);
123
124	KASSERT(psz->psz_owner == curlwp);
125	psz->psz_owner = NULL;
126	mutex_enter(&psz_lock);
127	psz_ev_excl.ev_count++;
128	mutex_exit(&psz_lock);
129}
130
131int
132pserialize_read_enter(void)
133{
134	int s;
135
136	s = splsoftserial();
137	curcpu()->ci_psz_read_depth++;
138	__insn_barrier();
139	return s;
140}
141
142void
143pserialize_read_exit(int s)
144{
145
146	KASSERT(kpreempt_disabled());
147
148	__insn_barrier();
149	if (__predict_false(curcpu()->ci_psz_read_depth-- == 0))
150		panic("mismatching pserialize_read_exit()");
151	splx(s);
152}
153
154/*
155 * pserialize_in_read_section:
156 *
157 *	True if the caller is in a pserialize read section.  To be used
158 *	only for diagnostic assertions where we want to guarantee the
159 *	condition like:
160 *
161 *		KASSERT(pserialize_in_read_section());
162 */
163bool
164pserialize_in_read_section(void)
165{
166
167	return kpreempt_disabled() && curcpu()->ci_psz_read_depth > 0;
168}
169
170/*
171 * pserialize_not_in_read_section:
172 *
173 *	True if the caller is not in a pserialize read section.  To be
174 *	used only for diagnostic assertions where we want to guarantee
175 *	the condition like:
176 *
177 *		KASSERT(pserialize_not_in_read_section());
178 */
179bool
180pserialize_not_in_read_section(void)
181{
182	bool notin;
183
184	kpreempt_disable();
185	notin = (curcpu()->ci_psz_read_depth == 0);
186	kpreempt_enable();
187
188	return notin;
189}
190