1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#pragma ident	"%Z%%M%	%I%	%E% SMI"
27
28#include <sys/types.h>
29#include <sys/param.h>
30#include <sys/stat.h>
31#include <sys/open.h>
32#include <sys/file.h>
33#include <sys/conf.h>
34#include <sys/modctl.h>
35#include <sys/cmn_err.h>
36#include <sys/bitmap.h>
37#include <sys/debug.h>
38#include <sys/kmem.h>
39#include <sys/errno.h>
40#include <sys/sysmacros.h>
41#include <sys/lockstat.h>
42#include <sys/atomic.h>
43#include <sys/dtrace.h>
44
45#include <sys/ddi.h>
46#include <sys/sunddi.h>
47
48typedef struct lockstat_probe {
49	const char	*lsp_func;
50	const char	*lsp_name;
51	int		lsp_probe;
52	dtrace_id_t	lsp_id;
53} lockstat_probe_t;
54
55lockstat_probe_t lockstat_probes[] =
56{
57	{ LS_MUTEX_ENTER,	LSA_ACQUIRE,	LS_MUTEX_ENTER_ACQUIRE },
58	{ LS_MUTEX_ENTER,	LSA_BLOCK,	LS_MUTEX_ENTER_BLOCK },
59	{ LS_MUTEX_ENTER,	LSA_SPIN,	LS_MUTEX_ENTER_SPIN },
60	{ LS_MUTEX_EXIT,	LSA_RELEASE,	LS_MUTEX_EXIT_RELEASE },
61	{ LS_MUTEX_DESTROY,	LSA_RELEASE,	LS_MUTEX_DESTROY_RELEASE },
62	{ LS_MUTEX_TRYENTER,	LSA_ACQUIRE,	LS_MUTEX_TRYENTER_ACQUIRE },
63	{ LS_LOCK_SET,		LSS_ACQUIRE,	LS_LOCK_SET_ACQUIRE },
64	{ LS_LOCK_SET,		LSS_SPIN,	LS_LOCK_SET_SPIN },
65	{ LS_LOCK_SET_SPL,	LSS_ACQUIRE,	LS_LOCK_SET_SPL_ACQUIRE },
66	{ LS_LOCK_SET_SPL,	LSS_SPIN,	LS_LOCK_SET_SPL_SPIN },
67	{ LS_LOCK_TRY,		LSS_ACQUIRE,	LS_LOCK_TRY_ACQUIRE },
68	{ LS_LOCK_CLEAR,	LSS_RELEASE,	LS_LOCK_CLEAR_RELEASE },
69	{ LS_LOCK_CLEAR_SPLX,	LSS_RELEASE,	LS_LOCK_CLEAR_SPLX_RELEASE },
70	{ LS_CLOCK_UNLOCK,	LSS_RELEASE,	LS_CLOCK_UNLOCK_RELEASE },
71	{ LS_RW_ENTER,		LSR_ACQUIRE,	LS_RW_ENTER_ACQUIRE },
72	{ LS_RW_ENTER,		LSR_BLOCK,	LS_RW_ENTER_BLOCK },
73	{ LS_RW_EXIT,		LSR_RELEASE,	LS_RW_EXIT_RELEASE },
74	{ LS_RW_TRYENTER,	LSR_ACQUIRE,	LS_RW_TRYENTER_ACQUIRE },
75	{ LS_RW_TRYUPGRADE,	LSR_UPGRADE,	LS_RW_TRYUPGRADE_UPGRADE },
76	{ LS_RW_DOWNGRADE,	LSR_DOWNGRADE,	LS_RW_DOWNGRADE_DOWNGRADE },
77	{ LS_THREAD_LOCK,	LST_SPIN,	LS_THREAD_LOCK_SPIN },
78	{ LS_THREAD_LOCK_HIGH,	LST_SPIN,	LS_THREAD_LOCK_HIGH_SPIN },
79	{ NULL }
80};
81
82static dev_info_t	*lockstat_devi;	/* saved in xxattach() for xxinfo() */
83static kmutex_t		lockstat_test;	/* for testing purposes only */
84static dtrace_provider_id_t lockstat_id;
85
86/*ARGSUSED*/
87static int
88lockstat_enable(void *arg, dtrace_id_t id, void *parg)
89{
90	lockstat_probe_t *probe = parg;
91
92	ASSERT(!lockstat_probemap[probe->lsp_probe]);
93
94	lockstat_probemap[probe->lsp_probe] = id;
95	membar_producer();
96
97	lockstat_hot_patch();
98	membar_producer();
99
100	/*
101	 * Immediately generate a record for the lockstat_test mutex
102	 * to verify that the mutex hot-patch code worked as expected.
103	 */
104	mutex_enter(&lockstat_test);
105	mutex_exit(&lockstat_test);
106	return (0);
107}
108
109/*ARGSUSED*/
110static void
111lockstat_disable(void *arg, dtrace_id_t id, void *parg)
112{
113	lockstat_probe_t *probe = parg;
114	int i;
115
116	ASSERT(lockstat_probemap[probe->lsp_probe]);
117
118	lockstat_probemap[probe->lsp_probe] = 0;
119	lockstat_hot_patch();
120	membar_producer();
121
122	/*
123	 * See if we have any probes left enabled.
124	 */
125	for (i = 0; i < LS_NPROBES; i++) {
126		if (lockstat_probemap[i]) {
127			/*
128			 * This probe is still enabled.  We don't need to deal
129			 * with waiting for all threads to be out of the
130			 * lockstat critical sections; just return.
131			 */
132			return;
133		}
134	}
135
136	/*
137	 * The delay() here isn't as cheesy as you might think.  We don't
138	 * want to busy-loop in the kernel, so we have to give up the
139	 * CPU between calls to lockstat_active_threads(); that much is
140	 * obvious.  But the reason it's a do..while loop rather than a
141	 * while loop is subtle.  The memory barrier above guarantees that
142	 * no threads will enter the lockstat code from this point forward.
143	 * However, another thread could already be executing lockstat code
144	 * without our knowledge if the update to its t_lockstat field hasn't
145	 * cleared its CPU's store buffer.  Delaying for one clock tick
146	 * guarantees that either (1) the thread will have *ample* time to
147	 * complete its work, or (2) the thread will be preempted, in which
148	 * case it will have to grab and release a dispatcher lock, which
149	 * will flush that CPU's store buffer.  Either way we're covered.
150	 */
151	do {
152		delay(1);
153	} while (lockstat_active_threads());
154}
155
156/*ARGSUSED*/
157static int
158lockstat_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
159{
160	return (0);
161}
162
163/* ARGSUSED */
164static int
165lockstat_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
166{
167	int error;
168
169	switch (infocmd) {
170	case DDI_INFO_DEVT2DEVINFO:
171		*result = (void *) lockstat_devi;
172		error = DDI_SUCCESS;
173		break;
174	case DDI_INFO_DEVT2INSTANCE:
175		*result = (void *)0;
176		error = DDI_SUCCESS;
177		break;
178	default:
179		error = DDI_FAILURE;
180	}
181	return (error);
182}
183
184/*ARGSUSED*/
185static void
186lockstat_provide(void *arg, const dtrace_probedesc_t *desc)
187{
188	int i = 0;
189
190	for (i = 0; lockstat_probes[i].lsp_func != NULL; i++) {
191		lockstat_probe_t *probe = &lockstat_probes[i];
192
193		if (dtrace_probe_lookup(lockstat_id, "genunix",
194		    probe->lsp_func, probe->lsp_name) != 0)
195			continue;
196
197		ASSERT(!probe->lsp_id);
198		probe->lsp_id = dtrace_probe_create(lockstat_id,
199		    "genunix", probe->lsp_func, probe->lsp_name,
200		    1, probe);
201	}
202}
203
204/*ARGSUSED*/
205static void
206lockstat_destroy(void *arg, dtrace_id_t id, void *parg)
207{
208	lockstat_probe_t *probe = parg;
209
210	ASSERT(!lockstat_probemap[probe->lsp_probe]);
211	probe->lsp_id = 0;
212}
213
214static dtrace_pattr_t lockstat_attr = {
215{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
216{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
217{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
218{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
219{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
220};
221
222static dtrace_pops_t lockstat_pops = {
223	lockstat_provide,
224	NULL,
225	lockstat_enable,
226	lockstat_disable,
227	NULL,
228	NULL,
229	NULL,
230	NULL,
231	NULL,
232	lockstat_destroy
233};
234
235static int
236lockstat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
237{
238	switch (cmd) {
239	case DDI_ATTACH:
240		break;
241	case DDI_RESUME:
242		return (DDI_SUCCESS);
243	default:
244		return (DDI_FAILURE);
245	}
246
247	if (ddi_create_minor_node(devi, "lockstat", S_IFCHR, 0,
248	    DDI_PSEUDO, 0) == DDI_FAILURE ||
249	    dtrace_register("lockstat", &lockstat_attr, DTRACE_PRIV_KERNEL,
250	    NULL, &lockstat_pops, NULL, &lockstat_id) != 0) {
251		ddi_remove_minor_node(devi, NULL);
252		return (DDI_FAILURE);
253	}
254
255	lockstat_probe = dtrace_probe;
256	membar_producer();
257
258	ddi_report_dev(devi);
259	lockstat_devi = devi;
260	return (DDI_SUCCESS);
261}
262
263static int
264lockstat_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
265{
266	switch (cmd) {
267	case DDI_DETACH:
268		break;
269	case DDI_SUSPEND:
270		return (DDI_SUCCESS);
271	default:
272		return (DDI_FAILURE);
273	}
274
275	if (dtrace_unregister(lockstat_id) != 0)
276		return (DDI_FAILURE);
277
278	ddi_remove_minor_node(devi, NULL);
279	return (DDI_SUCCESS);
280}
281
282/*
283 * Configuration data structures
284 */
285static struct cb_ops lockstat_cb_ops = {
286	lockstat_open,		/* open */
287	nodev,			/* close */
288	nulldev,		/* strategy */
289	nulldev,		/* print */
290	nodev,			/* dump */
291	nodev,			/* read */
292	nodev,			/* write */
293	nodev,			/* ioctl */
294	nodev,			/* devmap */
295	nodev,			/* mmap */
296	nodev,			/* segmap */
297	nochpoll,		/* poll */
298	ddi_prop_op,		/* cb_prop_op */
299	0,			/* streamtab */
300	D_MP | D_NEW		/* Driver compatibility flag */
301};
302
303static struct dev_ops lockstat_ops = {
304	DEVO_REV,		/* devo_rev, */
305	0,			/* refcnt */
306	lockstat_info,		/* getinfo */
307	nulldev,		/* identify */
308	nulldev,		/* probe */
309	lockstat_attach,	/* attach */
310	lockstat_detach,	/* detach */
311	nulldev,		/* reset */
312	&lockstat_cb_ops,	/* cb_ops */
313	NULL,			/* bus_ops */
314};
315
316static struct modldrv modldrv = {
317	&mod_driverops,		/* Type of module.  This one is a driver */
318	"Lock Statistics %I%",	/* name of module */
319	&lockstat_ops,		/* driver ops */
320};
321
322static struct modlinkage modlinkage = {
323	MODREV_1, (void *)&modldrv, NULL
324};
325
326int
327_init(void)
328{
329	return (mod_install(&modlinkage));
330}
331
332int
333_fini(void)
334{
335	return (mod_remove(&modlinkage));
336}
337
338int
339_info(struct modinfo *modinfop)
340{
341	return (mod_info(&modlinkage, modinfop));
342}
343