1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#include <sys/dtrace.h>
28#include <sys/cmn_err.h>
29#include <sys/tnf.h>
30#include <sys/atomic.h>
31#include <sys/prsystm.h>
32#include <sys/modctl.h>
33#include <sys/aio_impl.h>
34
35#ifdef __sparc
36#include <sys/privregs.h>
37#endif
38
39void (*dtrace_cpu_init)(processorid_t);
40void (*dtrace_modload)(struct modctl *);
41void (*dtrace_modunload)(struct modctl *);
42void (*dtrace_helpers_cleanup)(void);
43void (*dtrace_helpers_fork)(proc_t *, proc_t *);
44void (*dtrace_cpustart_init)(void);
45void (*dtrace_cpustart_fini)(void);
46void (*dtrace_cpc_fire)(uint64_t);
47
48void (*dtrace_debugger_init)(void);
49void (*dtrace_debugger_fini)(void);
50
51dtrace_vtime_state_t dtrace_vtime_active = 0;
52dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1;
53
54/*
55 * dtrace_cpc_in_use usage statement: this global variable is used by the cpc
56 * hardware overflow interrupt handler and the kernel cpc framework to check
57 * whether or not the DTrace cpc provider is currently in use. The variable is
58 * set before counters are enabled with the first enabling and cleared when
59 * the last enabling is disabled. Its value at any given time indicates the
60 * number of active dcpc based enablings. The global 'kcpc_cpuctx_lock' rwlock
61 * is held during initial setting to protect races between kcpc_open() and the
62 * first enabling. The locking provided by the DTrace subsystem, the kernel
63 * cpc framework and the cpu management framework protect consumers from race
64 * conditions on enabling and disabling probes.
65 */
66uint32_t dtrace_cpc_in_use = 0;
67
68typedef struct dtrace_hrestime {
69	lock_t		dthr_lock;		/* lock for this element */
70	timestruc_t	dthr_hrestime;		/* hrestime value */
71	int64_t		dthr_adj;		/* hrestime_adj value */
72	hrtime_t	dthr_hrtime;		/* hrtime value */
73} dtrace_hrestime_t;
74
75static dtrace_hrestime_t dtrace_hrestime[2];
76
77/*
78 * Making available adjustable high-resolution time in DTrace is regrettably
79 * more complicated than one might think it should be.  The problem is that
80 * the variables related to adjusted high-resolution time (hrestime,
81 * hrestime_adj and friends) are adjusted under hres_lock -- and this lock may
82 * be held when we enter probe context.  One might think that we could address
83 * this by having a single snapshot copy that is stored under a different lock
84 * from hres_tick(), using the snapshot iff hres_lock is locked in probe
85 * context.  Unfortunately, this too won't work:  because hres_lock is grabbed
86 * in more than just hres_tick() context, we could enter probe context
87 * concurrently on two different CPUs with both locks (hres_lock and the
88 * snapshot lock) held.  As this implies, the fundamental problem is that we
89 * need to have access to a snapshot of these variables that we _know_ will
90 * not be locked in probe context.  To effect this, we have two snapshots
91 * protected by two different locks, and we mandate that these snapshots are
92 * recorded in succession by a single thread calling dtrace_hres_tick().  (We
93 * assure this by calling it out of the same CY_HIGH_LEVEL cyclic that calls
94 * hres_tick().)  A single thread can't be in two places at once:  one of the
95 * snapshot locks is guaranteed to be unheld at all times.  The
96 * dtrace_gethrestime() algorithm is thus to check first one snapshot and then
97 * the other to find the unlocked snapshot.
98 */
99void
100dtrace_hres_tick(void)
101{
102	int i;
103	ushort_t spl;
104
105	for (i = 0; i < 2; i++) {
106		dtrace_hrestime_t tmp;
107
108		spl = hr_clock_lock();
109		tmp.dthr_hrestime = hrestime;
110		tmp.dthr_adj = hrestime_adj;
111		tmp.dthr_hrtime = dtrace_gethrtime();
112		hr_clock_unlock(spl);
113
114		lock_set(&dtrace_hrestime[i].dthr_lock);
115		dtrace_hrestime[i].dthr_hrestime = tmp.dthr_hrestime;
116		dtrace_hrestime[i].dthr_adj = tmp.dthr_adj;
117		dtrace_hrestime[i].dthr_hrtime = tmp.dthr_hrtime;
118		dtrace_membar_producer();
119
120		/*
121		 * To allow for lock-free examination of this lock, we use
122		 * the same trick that is used hres_lock; for more details,
123		 * see the description of this technique in sun4u/sys/clock.h.
124		 */
125		dtrace_hrestime[i].dthr_lock++;
126	}
127}
128
129hrtime_t
130dtrace_gethrestime(void)
131{
132	dtrace_hrestime_t snap;
133	hrtime_t now;
134	int i = 0, adj, nslt;
135
136	for (;;) {
137		snap.dthr_lock = dtrace_hrestime[i].dthr_lock;
138		dtrace_membar_consumer();
139		snap.dthr_hrestime = dtrace_hrestime[i].dthr_hrestime;
140		snap.dthr_hrtime = dtrace_hrestime[i].dthr_hrtime;
141		snap.dthr_adj = dtrace_hrestime[i].dthr_adj;
142		dtrace_membar_consumer();
143
144		if ((snap.dthr_lock & ~1) == dtrace_hrestime[i].dthr_lock)
145			break;
146
147		/*
148		 * If we're here, the lock was either locked, or it
149		 * transitioned while we were taking the snapshot.  Either
150		 * way, we're going to try the other dtrace_hrestime element;
151		 * we know that it isn't possible for both to be locked
152		 * simultaneously, so we will ultimately get a good snapshot.
153		 */
154		i ^= 1;
155	}
156
157	/*
158	 * We have a good snapshot.  Now perform any necessary adjustments.
159	 */
160	nslt = dtrace_gethrtime() - snap.dthr_hrtime;
161	ASSERT(nslt >= 0);
162
163	now = ((hrtime_t)snap.dthr_hrestime.tv_sec * (hrtime_t)NANOSEC) +
164	    snap.dthr_hrestime.tv_nsec;
165
166	if (snap.dthr_adj != 0) {
167		if (snap.dthr_adj > 0) {
168			adj = (nslt >> adj_shift);
169			if (adj > snap.dthr_adj)
170				adj = (int)snap.dthr_adj;
171		} else {
172			adj = -(nslt >> adj_shift);
173			if (adj < snap.dthr_adj)
174				adj = (int)snap.dthr_adj;
175		}
176		now += adj;
177	}
178
179	return (now);
180}
181
182void
183dtrace_vtime_enable(void)
184{
185	dtrace_vtime_state_t state, nstate;
186
187	do {
188		state = dtrace_vtime_active;
189
190		switch (state) {
191		case DTRACE_VTIME_INACTIVE:
192			nstate = DTRACE_VTIME_ACTIVE;
193			break;
194
195		case DTRACE_VTIME_INACTIVE_TNF:
196			nstate = DTRACE_VTIME_ACTIVE_TNF;
197			break;
198
199		case DTRACE_VTIME_ACTIVE:
200		case DTRACE_VTIME_ACTIVE_TNF:
201			panic("DTrace virtual time already enabled");
202			/*NOTREACHED*/
203		}
204
205	} while	(cas32((uint32_t *)&dtrace_vtime_active,
206	    state, nstate) != state);
207}
208
209void
210dtrace_vtime_disable(void)
211{
212	dtrace_vtime_state_t state, nstate;
213
214	do {
215		state = dtrace_vtime_active;
216
217		switch (state) {
218		case DTRACE_VTIME_ACTIVE:
219			nstate = DTRACE_VTIME_INACTIVE;
220			break;
221
222		case DTRACE_VTIME_ACTIVE_TNF:
223			nstate = DTRACE_VTIME_INACTIVE_TNF;
224			break;
225
226		case DTRACE_VTIME_INACTIVE:
227		case DTRACE_VTIME_INACTIVE_TNF:
228			panic("DTrace virtual time already disabled");
229			/*NOTREACHED*/
230		}
231
232	} while	(cas32((uint32_t *)&dtrace_vtime_active,
233	    state, nstate) != state);
234}
235
236void
237dtrace_vtime_enable_tnf(void)
238{
239	dtrace_vtime_state_t state, nstate;
240
241	do {
242		state = dtrace_vtime_active;
243
244		switch (state) {
245		case DTRACE_VTIME_ACTIVE:
246			nstate = DTRACE_VTIME_ACTIVE_TNF;
247			break;
248
249		case DTRACE_VTIME_INACTIVE:
250			nstate = DTRACE_VTIME_INACTIVE_TNF;
251			break;
252
253		case DTRACE_VTIME_ACTIVE_TNF:
254		case DTRACE_VTIME_INACTIVE_TNF:
255			panic("TNF already active");
256			/*NOTREACHED*/
257		}
258
259	} while	(cas32((uint32_t *)&dtrace_vtime_active,
260	    state, nstate) != state);
261}
262
263void
264dtrace_vtime_disable_tnf(void)
265{
266	dtrace_vtime_state_t state, nstate;
267
268	do {
269		state = dtrace_vtime_active;
270
271		switch (state) {
272		case DTRACE_VTIME_ACTIVE_TNF:
273			nstate = DTRACE_VTIME_ACTIVE;
274			break;
275
276		case DTRACE_VTIME_INACTIVE_TNF:
277			nstate = DTRACE_VTIME_INACTIVE;
278			break;
279
280		case DTRACE_VTIME_ACTIVE:
281		case DTRACE_VTIME_INACTIVE:
282			panic("TNF already inactive");
283			/*NOTREACHED*/
284		}
285
286	} while	(cas32((uint32_t *)&dtrace_vtime_active,
287	    state, nstate) != state);
288}
289
290void
291dtrace_vtime_switch(kthread_t *next)
292{
293	dtrace_icookie_t cookie;
294	hrtime_t ts;
295
296	if (tnf_tracing_active) {
297		tnf_thread_switch(next);
298
299		if (dtrace_vtime_active == DTRACE_VTIME_INACTIVE_TNF)
300			return;
301	}
302
303	cookie = dtrace_interrupt_disable();
304	ts = dtrace_gethrtime();
305
306	if (curthread->t_dtrace_start != 0) {
307		curthread->t_dtrace_vtime += ts - curthread->t_dtrace_start;
308		curthread->t_dtrace_start = 0;
309	}
310
311	next->t_dtrace_start = ts;
312
313	dtrace_interrupt_enable(cookie);
314}
315
316void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *);
317void (*dtrace_fasttrap_exec_ptr)(proc_t *);
318void (*dtrace_fasttrap_exit_ptr)(proc_t *);
319
320/*
321 * This function is called by cfork() in the event that it appears that
322 * there may be dtrace tracepoints active in the parent process's address
323 * space. This first confirms the existence of dtrace tracepoints in the
324 * parent process and calls into the fasttrap module to remove the
325 * corresponding tracepoints from the child. By knowing that there are
326 * existing tracepoints, and ensuring they can't be removed, we can rely
327 * on the fasttrap module remaining loaded.
328 */
329void
330dtrace_fasttrap_fork(proc_t *p, proc_t *cp)
331{
332	ASSERT(p->p_proc_flag & P_PR_LOCK);
333	ASSERT(p->p_dtrace_count > 0);
334	ASSERT(dtrace_fasttrap_fork_ptr != NULL);
335
336	dtrace_fasttrap_fork_ptr(p, cp);
337}
338