1/*-
2 * Copyright (c) 1996 Bruce D. Evans.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD$");
29
30#ifdef GUPROF
31#include "opt_i586_guprof.h"
32#include "opt_perfmon.h"
33
34#include <sys/param.h>
35#include <sys/systm.h>
36#include <sys/bus.h>
37#include <sys/cpu.h>
38#include <sys/eventhandler.h>
39#include <sys/gmon.h>
40#include <sys/kernel.h>
41#include <sys/smp.h>
42#include <sys/sysctl.h>
43
44#include <machine/clock.h>
45#include <machine/perfmon.h>
46#include <machine/timerreg.h>
47
48#define	CPUTIME_CLOCK_UNINITIALIZED	0
49#define	CPUTIME_CLOCK_I8254		1
50#define	CPUTIME_CLOCK_TSC		2
51#define	CPUTIME_CLOCK_I586_PMC		3
52#define	CPUTIME_CLOCK_I8254_SHIFT	7
53
54int	cputime_bias = 1;	/* initialize for locality of reference */
55
56static int	cputime_clock = CPUTIME_CLOCK_UNINITIALIZED;
57#if defined(PERFMON) && defined(I586_PMC_GUPROF)
58static u_int	cputime_clock_pmc_conf = I586_PMC_GUPROF;
59static int	cputime_clock_pmc_init;
60static struct gmonparam saved_gmp;
61#endif
62#if defined(I586_CPU) || defined(I686_CPU)
63static int	cputime_prof_active;
64#endif
65#endif /* GUPROF */
66
67#ifdef __GNUCLIKE_ASM
68__asm("								\n\
69GM_STATE	=	0					\n\
70GMON_PROF_OFF	=	3					\n\
71								\n\
72	.text							\n\
73	.p2align 4,0x90						\n\
74	.globl	__mcount					\n\
75	.type	__mcount,@function				\n\
76__mcount:							\n\
77	#							\n\
78	# Check that we are profiling.  Do it early for speed.	\n\
79	#							\n\
80	cmpl	$GMON_PROF_OFF,_gmonparam+GM_STATE		\n\
81	je	.mcount_exit					\n\
82	#							\n\
83	# __mcount is the same as [.]mcount except the caller	\n\
84	# hasn't changed the stack except to call here, so the	\n\
85	# caller's raddr is above our raddr.			\n\
86	#							\n\
87	movl	4(%esp),%edx					\n\
88	jmp	.got_frompc					\n\
89								\n\
90	.p2align 4,0x90						\n\
91	.globl	.mcount						\n\
92.mcount:							\n\
93	cmpl	$GMON_PROF_OFF,_gmonparam+GM_STATE		\n\
94	je	.mcount_exit					\n\
95	#							\n\
96	# The caller's stack frame has already been built, so	\n\
97	# %ebp is the caller's frame pointer.  The caller's	\n\
98	# raddr is in the caller's frame following the caller's	\n\
99	# caller's frame pointer.				\n\
100	#							\n\
101	movl	4(%ebp),%edx					\n\
102.got_frompc:							\n\
103	#							\n\
104	# Our raddr is the caller's pc.				\n\
105	#							\n\
106	movl	(%esp),%eax					\n\
107								\n\
108	pushfl							\n\
109	pushl	%eax						\n\
110	pushl	%edx						\n\
111	cli							\n\
112	call	mcount						\n\
113	addl	$8,%esp						\n\
114	popfl							\n\
115.mcount_exit:							\n\
116	ret	$0						\n\
117");
118#else /* !__GNUCLIKE_ASM */
119#error "this file needs to be ported to your compiler"
120#endif /* __GNUCLIKE_ASM */
121
122#ifdef GUPROF
123/*
124 * [.]mexitcount saves the return register(s), loads selfpc and calls
125 * mexitcount(selfpc) to do the work.  Someday it should be in a machine
126 * dependent file together with cputime(), __mcount and [.]mcount.  cputime()
127 * can't just be put in machdep.c because it has to be compiled without -pg.
128 */
129#ifdef __GNUCLIKE_ASM
130__asm("								\n\
131	.text							\n\
132#								\n\
133# Dummy label to be seen when gprof -u hides [.]mexitcount.	\n\
134#								\n\
135	.p2align 4,0x90						\n\
136	.globl	__mexitcount					\n\
137	.type	__mexitcount,@function				\n\
138__mexitcount:							\n\
139	nop							\n\
140								\n\
141GMON_PROF_HIRES	=	4					\n\
142								\n\
143	.p2align 4,0x90						\n\
144	.globl	.mexitcount					\n\
145.mexitcount:							\n\
146	cmpl	$GMON_PROF_HIRES,_gmonparam+GM_STATE		\n\
147	jne	.mexitcount_exit				\n\
148	pushl	%edx						\n\
149	pushl	%eax						\n\
150	movl	8(%esp),%eax					\n\
151	pushfl							\n\
152	pushl	%eax						\n\
153	cli							\n\
154	call	mexitcount					\n\
155	addl	$4,%esp						\n\
156	popfl							\n\
157	popl	%eax						\n\
158	popl	%edx						\n\
159.mexitcount_exit:						\n\
160	ret	$0						\n\
161");
162#endif /* __GNUCLIKE_ASM */
163
164/*
165 * Return the time elapsed since the last call.  The units are machine-
166 * dependent.
167 */
168int
169cputime()
170{
171	u_int count;
172	int delta;
173#if (defined(I586_CPU) || defined(I686_CPU)) && \
174    defined(PERFMON) && defined(I586_PMC_GUPROF) && !defined(SMP)
175	u_quad_t event_count;
176#endif
177	u_char high, low;
178	static u_int prev_count;
179
180#if defined(I586_CPU) || defined(I686_CPU)
181	if (cputime_clock == CPUTIME_CLOCK_TSC) {
182		/*
183		 * Scale the TSC a little to make cputime()'s frequency
184		 * fit in an int, assuming that the TSC frequency fits
185		 * in a u_int.  Use a fixed scale since dynamic scaling
186		 * would be slower and we can't really use the low bit
187		 * of precision.
188		 */
189		count = (u_int)rdtsc() & ~1u;
190		delta = (int)(count - prev_count) >> 1;
191		prev_count = count;
192		return (delta);
193	}
194#if defined(PERFMON) && defined(I586_PMC_GUPROF) && !defined(SMP)
195	if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
196		/*
197		 * XXX permon_read() should be inlined so that the
198		 * perfmon module doesn't need to be compiled with
199		 * profiling disabled and so that it is fast.
200		 */
201		perfmon_read(0, &event_count);
202
203		count = (u_int)event_count;
204		delta = (int)(count - prev_count);
205		prev_count = count;
206		return (delta);
207	}
208#endif /* PERFMON && I586_PMC_GUPROF && !SMP */
209#endif /* I586_CPU || I686_CPU */
210
211	/*
212	 * Read the current value of the 8254 timer counter 0.
213	 */
214	outb(TIMER_MODE, TIMER_SEL0 | TIMER_LATCH);
215	low = inb(TIMER_CNTR0);
216	high = inb(TIMER_CNTR0);
217	count = ((high << 8) | low) << CPUTIME_CLOCK_I8254_SHIFT;
218
219	/*
220	 * The timer counts down from TIMER_CNTR0_MAX to 0 and then resets.
221	 * While profiling is enabled, this routine is called at least twice
222	 * per timer reset (for mcounting and mexitcounting hardclock()),
223	 * so at most one reset has occurred since the last call, and one
224	 * has occurred iff the current count is larger than the previous
225	 * count.  This allows counter underflow to be detected faster
226	 * than in microtime().
227	 */
228	delta = prev_count - count;
229	prev_count = count;
230	if ((int) delta <= 0)
231		return (delta + (i8254_max_count << CPUTIME_CLOCK_I8254_SHIFT));
232	return (delta);
233}
234
235static int
236sysctl_machdep_cputime_clock(SYSCTL_HANDLER_ARGS)
237{
238	int clock;
239	int error;
240#if defined(PERFMON) && defined(I586_PMC_GUPROF)
241	int event;
242	struct pmc pmc;
243#endif
244
245	clock = cputime_clock;
246#if defined(PERFMON) && defined(I586_PMC_GUPROF)
247	if (clock == CPUTIME_CLOCK_I586_PMC) {
248		pmc.pmc_val = cputime_clock_pmc_conf;
249		clock += pmc.pmc_event;
250	}
251#endif
252	error = sysctl_handle_opaque(oidp, &clock, sizeof clock, req);
253	if (error == 0 && req->newptr != NULL) {
254#if defined(PERFMON) && defined(I586_PMC_GUPROF)
255		if (clock >= CPUTIME_CLOCK_I586_PMC) {
256			event = clock - CPUTIME_CLOCK_I586_PMC;
257			if (event >= 256)
258				return (EINVAL);
259			pmc.pmc_num = 0;
260			pmc.pmc_event = event;
261			pmc.pmc_unit = 0;
262			pmc.pmc_flags = PMCF_E | PMCF_OS | PMCF_USR;
263			pmc.pmc_mask = 0;
264			cputime_clock_pmc_conf = pmc.pmc_val;
265			cputime_clock = CPUTIME_CLOCK_I586_PMC;
266		} else
267#endif
268		{
269			if (clock < 0 || clock >= CPUTIME_CLOCK_I586_PMC)
270				return (EINVAL);
271			cputime_clock = clock;
272		}
273	}
274	return (error);
275}
276
277SYSCTL_PROC(_machdep, OID_AUTO, cputime_clock, CTLTYPE_INT | CTLFLAG_RW,
278	    0, sizeof(u_int), sysctl_machdep_cputime_clock, "I", "");
279
280/*
281 * The start and stop routines need not be here since we turn off profiling
282 * before calling them.  They are here for convenience.
283 */
284
285void
286startguprof(gp)
287	struct gmonparam *gp;
288{
289#if defined(I586_CPU) || defined(I686_CPU)
290	uint64_t freq;
291
292	freq = atomic_load_acq_64(&tsc_freq);
293	if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) {
294		if (freq != 0 && mp_ncpus == 1)
295			cputime_clock = CPUTIME_CLOCK_TSC;
296		else
297			cputime_clock = CPUTIME_CLOCK_I8254;
298	}
299	if (cputime_clock == CPUTIME_CLOCK_TSC) {
300		gp->profrate = freq >> 1;
301		cputime_prof_active = 1;
302	} else
303		gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
304#if defined(PERFMON) && defined(I586_PMC_GUPROF)
305	if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
306		if (perfmon_avail() &&
307		    perfmon_setup(0, cputime_clock_pmc_conf) == 0) {
308			if (perfmon_start(0) != 0)
309				perfmon_fini(0);
310			else {
311				/* XXX 1 event == 1 us. */
312				gp->profrate = 1000000;
313
314				saved_gmp = *gp;
315
316				/* Zap overheads.  They are invalid. */
317				gp->cputime_overhead = 0;
318				gp->mcount_overhead = 0;
319				gp->mcount_post_overhead = 0;
320				gp->mcount_pre_overhead = 0;
321				gp->mexitcount_overhead = 0;
322				gp->mexitcount_post_overhead = 0;
323				gp->mexitcount_pre_overhead = 0;
324
325				cputime_clock_pmc_init = TRUE;
326			}
327		}
328	}
329#endif /* PERFMON && I586_PMC_GUPROF */
330#else /* !(I586_CPU || I686_CPU) */
331	if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED)
332		cputime_clock = CPUTIME_CLOCK_I8254;
333	gp->profrate = i8254_freq << CPUTIME_CLOCK_I8254_SHIFT;
334#endif /* I586_CPU || I686_CPU */
335	cputime_bias = 0;
336	cputime();
337}
338
339void
340stopguprof(gp)
341	struct gmonparam *gp;
342{
343#if defined(PERFMON) && defined(I586_PMC_GUPROF)
344	if (cputime_clock_pmc_init) {
345		*gp = saved_gmp;
346		perfmon_fini(0);
347		cputime_clock_pmc_init = FALSE;
348	}
349#endif
350#if defined(I586_CPU) || defined(I686_CPU)
351	if (cputime_clock == CPUTIME_CLOCK_TSC)
352		cputime_prof_active = 0;
353#endif
354}
355
356#if defined(I586_CPU) || defined(I686_CPU)
357/* If the cpu frequency changed while profiling, report a warning. */
358static void
359tsc_freq_changed(void *arg, const struct cf_level *level, int status)
360{
361
362	/*
363	 * If there was an error during the transition or
364	 * TSC is P-state invariant, don't do anything.
365	 */
366	if (status != 0 || tsc_is_invariant)
367		return;
368	if (cputime_prof_active && cputime_clock == CPUTIME_CLOCK_TSC)
369		printf("warning: cpu freq changed while profiling active\n");
370}
371
372EVENTHANDLER_DEFINE(cpufreq_post_change, tsc_freq_changed, NULL,
373    EVENTHANDLER_PRI_ANY);
374#endif /* I586_CPU || I686_CPU */
375
376#endif /* GUPROF */
377