1/*-
2 * Copyright (c) 1996 Bruce D. Evans.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD$");
29
30#ifdef GUPROF
31#if 0
32#include "opt_i586_guprof.h"
33#include "opt_perfmon.h"
34#endif
35
36#include <sys/param.h>
37#include <sys/systm.h>
38#include <sys/bus.h>
39#include <sys/cpu.h>
40#include <sys/eventhandler.h>
41#include <sys/gmon.h>
42#include <sys/kernel.h>
43#include <sys/smp.h>
44#include <sys/sysctl.h>
45
46#include <machine/clock.h>
47#if 0
48#include <machine/perfmon.h>
49#endif
50#include <machine/timerreg.h>
51
52#define	CPUTIME_CLOCK_UNINITIALIZED	0
53#define	CPUTIME_CLOCK_I8254		1
54#define	CPUTIME_CLOCK_TSC		2
55#define	CPUTIME_CLOCK_I586_PMC		3
56#define	CPUTIME_CLOCK_I8254_SHIFT	7
57
58int	cputime_bias = 1;	/* initialize for locality of reference */
59
60static int	cputime_clock = CPUTIME_CLOCK_UNINITIALIZED;
61#if defined(PERFMON) && defined(I586_PMC_GUPROF)
62static u_int	cputime_clock_pmc_conf = I586_PMC_GUPROF;
63static int	cputime_clock_pmc_init;
64static struct gmonparam saved_gmp;
65#endif
66static int	cputime_prof_active;
67#endif /* GUPROF */
68
69#ifdef __GNUCLIKE_ASM
70__asm("								\n\
71GM_STATE	=	0					\n\
72GMON_PROF_OFF	=	3					\n\
73								\n\
74	.text							\n\
75	.p2align 4,0x90						\n\
76	.globl	__mcount					\n\
77	.type	__mcount,@function				\n\
78__mcount:							\n\
79	#							\n\
80	# Check that we are profiling.  Do it early for speed.	\n\
81	#							\n\
82	cmpl	$GMON_PROF_OFF,_gmonparam+GM_STATE		\n\
83	je	.mcount_exit					\n\
84	#							\n\
85	# __mcount is the same as [.]mcount except the caller	\n\
86	# hasn't changed the stack except to call here, so the	\n\
87	# caller's raddr is above our raddr.			\n\
88	#							\n\
89	pushq	%rax						\n\
90	pushq	%rdx						\n\
91	pushq	%rcx						\n\
92	pushq	%rsi						\n\
93	pushq	%rdi						\n\
94	pushq	%r8						\n\
95	pushq	%r9						\n\
96	movq	7*8+8(%rsp),%rdi				\n\
97	jmp	.got_frompc					\n\
98								\n\
99	.p2align 4,0x90						\n\
100	.globl	.mcount						\n\
101.mcount:							\n\
102	cmpl	$GMON_PROF_OFF,_gmonparam+GM_STATE		\n\
103	je	.mcount_exit					\n\
104	#							\n\
105	# The caller's stack frame has already been built, so	\n\
106	# %rbp is the caller's frame pointer.  The caller's	\n\
107	# raddr is in the caller's frame following the caller's	\n\
108	# caller's frame pointer.				\n\
109	#							\n\
110	pushq	%rax						\n\
111	pushq	%rdx						\n\
112	pushq	%rcx						\n\
113	pushq	%rsi						\n\
114	pushq	%rdi						\n\
115	pushq	%r8						\n\
116	pushq	%r9						\n\
117	movq	8(%rbp),%rdi					\n\
118.got_frompc:							\n\
119	#							\n\
120	# Our raddr is the caller's pc.				\n\
121	#							\n\
122	movq	7*8(%rsp),%rsi					\n\
123								\n\
124	pushfq							\n\
125	cli							\n\
126	call	mcount						\n\
127	popfq							\n\
128	popq	%r9						\n\
129	popq	%r8						\n\
130	popq	%rdi						\n\
131	popq	%rsi						\n\
132	popq	%rcx						\n\
133	popq	%rdx						\n\
134	popq	%rax						\n\
135.mcount_exit:							\n\
136	ret	$0						\n\
137");
138#else /* !__GNUCLIKE_ASM */
139#error "this file needs to be ported to your compiler"
140#endif /* __GNUCLIKE_ASM */
141
142#ifdef GUPROF
143/*
144 * [.]mexitcount saves the return register(s), loads selfpc and calls
145 * mexitcount(selfpc) to do the work.  Someday it should be in a machine
146 * dependent file together with cputime(), __mcount and [.]mcount.  cputime()
147 * can't just be put in machdep.c because it has to be compiled without -pg.
148 */
149#ifdef __GNUCLIKE_ASM
150__asm("								\n\
151	.text							\n\
152#								\n\
153# Dummy label to be seen when gprof -u hides [.]mexitcount.	\n\
154#								\n\
155	.p2align 4,0x90						\n\
156	.globl	__mexitcount					\n\
157	.type	__mexitcount,@function				\n\
158__mexitcount:							\n\
159	nop							\n\
160								\n\
161GMON_PROF_HIRES	=	4					\n\
162								\n\
163	.p2align 4,0x90						\n\
164	.globl	.mexitcount					\n\
165.mexitcount:							\n\
166	cmpl	$GMON_PROF_HIRES,_gmonparam+GM_STATE		\n\
167	jne	.mexitcount_exit				\n\
168	pushq	%rax						\n\
169	pushq	%rdx						\n\
170	pushq	%rcx						\n\
171	pushq	%rsi						\n\
172	pushq	%rdi						\n\
173	pushq	%r8						\n\
174	pushq	%r9						\n\
175	movq	7*8(%rsp),%rdi					\n\
176	pushfq							\n\
177	cli							\n\
178	call	mexitcount					\n\
179	popfq							\n\
180	popq	%r9						\n\
181	popq	%r8						\n\
182	popq	%rdi						\n\
183	popq	%rsi						\n\
184	popq	%rcx						\n\
185	popq	%rdx						\n\
186	popq	%rax						\n\
187.mexitcount_exit:						\n\
188	ret	$0						\n\
189");
190#endif /* __GNUCLIKE_ASM */
191
192/*
193 * Return the time elapsed since the last call.  The units are machine-
194 * dependent.
195 */
196int
197cputime()
198{
199	u_int count;
200	int delta;
201#if defined(PERFMON) && defined(I586_PMC_GUPROF) && !defined(SMP)
202	u_quad_t event_count;
203#endif
204	u_char high, low;
205	static u_int prev_count;
206
207	if (cputime_clock == CPUTIME_CLOCK_TSC) {
208		/*
209		 * Scale the TSC a little to make cputime()'s frequency
210		 * fit in an int, assuming that the TSC frequency fits
211		 * in a u_int.  Use a fixed scale since dynamic scaling
212		 * would be slower and we can't really use the low bit
213		 * of precision.
214		 */
215		count = (u_int)rdtsc() & ~1u;
216		delta = (int)(count - prev_count) >> 1;
217		prev_count = count;
218		return (delta);
219	}
220#if defined(PERFMON) && defined(I586_PMC_GUPROF) && !defined(SMP)
221	if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
222		/*
223		 * XXX permon_read() should be inlined so that the
224		 * perfmon module doesn't need to be compiled with
225		 * profiling disabled and so that it is fast.
226		 */
227		perfmon_read(0, &event_count);
228
229		count = (u_int)event_count;
230		delta = (int)(count - prev_count);
231		prev_count = count;
232		return (delta);
233	}
234#endif /* PERFMON && I586_PMC_GUPROF && !SMP */
235
236	/*
237	 * Read the current value of the 8254 timer counter 0.
238	 */
239	outb(TIMER_MODE, TIMER_SEL0 | TIMER_LATCH);
240	low = inb(TIMER_CNTR0);
241	high = inb(TIMER_CNTR0);
242	count = ((high << 8) | low) << CPUTIME_CLOCK_I8254_SHIFT;
243
244	/*
245	 * The timer counts down from TIMER_CNTR0_MAX to 0 and then resets.
246	 * While profiling is enabled, this routine is called at least twice
247	 * per timer reset (for mcounting and mexitcounting hardclock()),
248	 * so at most one reset has occurred since the last call, and one
249	 * has occurred iff the current count is larger than the previous
250	 * count.  This allows counter underflow to be detected faster
251	 * than in microtime().
252	 */
253	delta = prev_count - count;
254	prev_count = count;
255	if ((int) delta <= 0)
256		return (delta + (i8254_max_count << CPUTIME_CLOCK_I8254_SHIFT));
257	return (delta);
258}
259
260static int
261sysctl_machdep_cputime_clock(SYSCTL_HANDLER_ARGS)
262{
263	int clock;
264	int error;
265#if defined(PERFMON) && defined(I586_PMC_GUPROF)
266	int event;
267	struct pmc pmc;
268#endif
269
270	clock = cputime_clock;
271#if defined(PERFMON) && defined(I586_PMC_GUPROF)
272	if (clock == CPUTIME_CLOCK_I586_PMC) {
273		pmc.pmc_val = cputime_clock_pmc_conf;
274		clock += pmc.pmc_event;
275	}
276#endif
277	error = sysctl_handle_opaque(oidp, &clock, sizeof clock, req);
278	if (error == 0 && req->newptr != NULL) {
279#if defined(PERFMON) && defined(I586_PMC_GUPROF)
280		if (clock >= CPUTIME_CLOCK_I586_PMC) {
281			event = clock - CPUTIME_CLOCK_I586_PMC;
282			if (event >= 256)
283				return (EINVAL);
284			pmc.pmc_num = 0;
285			pmc.pmc_event = event;
286			pmc.pmc_unit = 0;
287			pmc.pmc_flags = PMCF_E | PMCF_OS | PMCF_USR;
288			pmc.pmc_mask = 0;
289			cputime_clock_pmc_conf = pmc.pmc_val;
290			cputime_clock = CPUTIME_CLOCK_I586_PMC;
291		} else
292#endif
293		{
294			if (clock < 0 || clock >= CPUTIME_CLOCK_I586_PMC)
295				return (EINVAL);
296			cputime_clock = clock;
297		}
298	}
299	return (error);
300}
301
302SYSCTL_PROC(_machdep, OID_AUTO, cputime_clock, CTLTYPE_INT | CTLFLAG_RW,
303	    0, sizeof(u_int), sysctl_machdep_cputime_clock, "I", "");
304
305/*
306 * The start and stop routines need not be here since we turn off profiling
307 * before calling them.  They are here for convenience.
308 */
309
310void
311startguprof(gp)
312	struct gmonparam *gp;
313{
314	uint64_t freq;
315
316	freq = atomic_load_acq_64(&tsc_freq);
317	if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) {
318		if (freq != 0 && mp_ncpus == 1)
319			cputime_clock = CPUTIME_CLOCK_TSC;
320		else
321			cputime_clock = CPUTIME_CLOCK_I8254;
322	}
323	if (cputime_clock == CPUTIME_CLOCK_TSC) {
324		gp->profrate = freq >> 1;
325		cputime_prof_active = 1;
326	} else
327		gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
328#if defined(PERFMON) && defined(I586_PMC_GUPROF)
329	if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
330		if (perfmon_avail() &&
331		    perfmon_setup(0, cputime_clock_pmc_conf) == 0) {
332			if (perfmon_start(0) != 0)
333				perfmon_fini(0);
334			else {
335				/* XXX 1 event == 1 us. */
336				gp->profrate = 1000000;
337
338				saved_gmp = *gp;
339
340				/* Zap overheads.  They are invalid. */
341				gp->cputime_overhead = 0;
342				gp->mcount_overhead = 0;
343				gp->mcount_post_overhead = 0;
344				gp->mcount_pre_overhead = 0;
345				gp->mexitcount_overhead = 0;
346				gp->mexitcount_post_overhead = 0;
347				gp->mexitcount_pre_overhead = 0;
348
349				cputime_clock_pmc_init = TRUE;
350			}
351		}
352	}
353#endif /* PERFMON && I586_PMC_GUPROF */
354	cputime_bias = 0;
355	cputime();
356}
357
358void
359stopguprof(gp)
360	struct gmonparam *gp;
361{
362#if defined(PERFMON) && defined(I586_PMC_GUPROF)
363	if (cputime_clock_pmc_init) {
364		*gp = saved_gmp;
365		perfmon_fini(0);
366		cputime_clock_pmc_init = FALSE;
367	}
368#endif
369	if (cputime_clock == CPUTIME_CLOCK_TSC)
370		cputime_prof_active = 0;
371}
372
373/* If the cpu frequency changed while profiling, report a warning. */
374static void
375tsc_freq_changed(void *arg, const struct cf_level *level, int status)
376{
377
378	/*
379	 * If there was an error during the transition or
380	 * TSC is P-state invariant, don't do anything.
381	 */
382	if (status != 0 || tsc_is_invariant)
383		return;
384	if (cputime_prof_active && cputime_clock == CPUTIME_CLOCK_TSC)
385		printf("warning: cpu freq changed while profiling active\n");
386}
387
388EVENTHANDLER_DEFINE(cpufreq_post_change, tsc_freq_changed, NULL,
389    EVENTHANDLER_PRI_ANY);
390
391#endif /* GUPROF */
392