1#include <linux/kernel.h>
2#include <linux/sched.h>
3#include <linux/interrupt.h>
4#include <linux/init.h>
5#include <linux/clocksource.h>
6#include <linux/time.h>
7#include <linux/acpi.h>
8#include <linux/cpufreq.h>
9
10#include <asm/timex.h>
11
12static int notsc __initdata = 0;
13
14unsigned int cpu_khz;		/* TSC clocks / usec, not used here */
15EXPORT_SYMBOL(cpu_khz);
16unsigned int tsc_khz;
17EXPORT_SYMBOL(tsc_khz);
18
19static unsigned int cyc2ns_scale __read_mostly;
20
21void set_cyc2ns_scale(unsigned long khz)
22{
23	cyc2ns_scale = (NSEC_PER_MSEC << NS_SCALE) / khz;
24}
25
26static unsigned long long cycles_2_ns(unsigned long long cyc)
27{
28	return (cyc * cyc2ns_scale) >> NS_SCALE;
29}
30
31unsigned long long sched_clock(void)
32{
33	unsigned long a = 0;
34
35	/* Could do CPU core sync here. Opteron can execute rdtsc speculatively,
36	 * which means it is not completely exact and may not be monotonous
37	 * between CPUs. But the errors should be too small to matter for
38	 * scheduling purposes.
39	 */
40
41	rdtscll(a);
42	return cycles_2_ns(a);
43}
44
45static int tsc_unstable;
46
47static inline int check_tsc_unstable(void)
48{
49	return tsc_unstable;
50}
51#ifdef CONFIG_CPU_FREQ
52
53/* Frequency scaling support. Adjust the TSC based timer when the cpu frequency
54 * changes.
55 *
56 * RED-PEN: On SMP we assume all CPUs run with the same frequency.  It's
57 * not that important because current Opteron setups do not support
58 * scaling on SMP anyroads.
59 *
60 * Should fix up last_tsc too. Currently gettimeofday in the
61 * first tick after the change will be slightly wrong.
62 */
63
64#include <linux/workqueue.h>
65
66static unsigned int cpufreq_delayed_issched = 0;
67static unsigned int cpufreq_init = 0;
68static struct work_struct cpufreq_delayed_get_work;
69
70static void handle_cpufreq_delayed_get(struct work_struct *v)
71{
72	unsigned int cpu;
73	for_each_online_cpu(cpu) {
74		cpufreq_get(cpu);
75	}
76	cpufreq_delayed_issched = 0;
77}
78
79static unsigned int  ref_freq = 0;
80static unsigned long loops_per_jiffy_ref = 0;
81
82static unsigned long tsc_khz_ref = 0;
83
84static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
85				 void *data)
86{
87	struct cpufreq_freqs *freq = data;
88	unsigned long *lpj, dummy;
89
90	if (cpu_has(&cpu_data[freq->cpu], X86_FEATURE_CONSTANT_TSC))
91		return 0;
92
93	lpj = &dummy;
94	if (!(freq->flags & CPUFREQ_CONST_LOOPS))
95#ifdef CONFIG_SMP
96		lpj = &cpu_data[freq->cpu].loops_per_jiffy;
97#else
98		lpj = &boot_cpu_data.loops_per_jiffy;
99#endif
100
101	if (!ref_freq) {
102		ref_freq = freq->old;
103		loops_per_jiffy_ref = *lpj;
104		tsc_khz_ref = tsc_khz;
105	}
106	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
107		(val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
108		(val == CPUFREQ_RESUMECHANGE)) {
109		*lpj =
110		cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
111
112		tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);
113		if (!(freq->flags & CPUFREQ_CONST_LOOPS))
114			mark_tsc_unstable("cpufreq changes");
115	}
116
117	set_cyc2ns_scale(tsc_khz_ref);
118
119	return 0;
120}
121
122static struct notifier_block time_cpufreq_notifier_block = {
123	.notifier_call  = time_cpufreq_notifier
124};
125
126static int __init cpufreq_tsc(void)
127{
128	INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get);
129	if (!cpufreq_register_notifier(&time_cpufreq_notifier_block,
130				       CPUFREQ_TRANSITION_NOTIFIER))
131		cpufreq_init = 1;
132	return 0;
133}
134
135core_initcall(cpufreq_tsc);
136
137#endif
138
139static int tsc_unstable = 0;
140
141/*
142 * Make an educated guess if the TSC is trustworthy and synchronized
143 * over all CPUs.
144 */
145__cpuinit int unsynchronized_tsc(void)
146{
147	if (tsc_unstable)
148		return 1;
149
150#ifdef CONFIG_SMP
151	if (apic_is_clustered_box())
152		return 1;
153#endif
154	/* Most intel systems have synchronized TSCs except for
155	   multi node systems */
156 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
157#ifdef CONFIG_ACPI
158		/* But TSC doesn't tick in C3 so don't use it there */
159		if (acpi_gbl_FADT.header.length > 0 && acpi_gbl_FADT.C3latency < 1000)
160			return 1;
161#endif
162 		return 0;
163	}
164
165 	/* Assume multi socket systems are not synchronized */
166 	return num_present_cpus() > 1;
167}
168
169int __init notsc_setup(char *s)
170{
171	notsc = 1;
172	return 1;
173}
174
175__setup("notsc", notsc_setup);
176
177
178/* clock source code: */
179static cycle_t read_tsc(void)
180{
181	cycle_t ret = (cycle_t)get_cycles_sync();
182	return ret;
183}
184
185static cycle_t __vsyscall_fn vread_tsc(void)
186{
187	cycle_t ret = (cycle_t)get_cycles_sync();
188	return ret;
189}
190
191static struct clocksource clocksource_tsc = {
192	.name			= "tsc",
193	.rating			= 300,
194	.read			= read_tsc,
195	.mask			= CLOCKSOURCE_MASK(64),
196	.shift			= 22,
197	.flags			= CLOCK_SOURCE_IS_CONTINUOUS |
198				  CLOCK_SOURCE_MUST_VERIFY,
199	.vread			= vread_tsc,
200};
201
202void mark_tsc_unstable(char *reason)
203{
204	if (!tsc_unstable) {
205		tsc_unstable = 1;
206		printk("Marking TSC unstable due to %s\n", reason);
207		/* Change only the rating, when not registered */
208		if (clocksource_tsc.mult)
209			clocksource_change_rating(&clocksource_tsc, 0);
210		else
211			clocksource_tsc.rating = 0;
212	}
213}
214EXPORT_SYMBOL_GPL(mark_tsc_unstable);
215
216void __init init_tsc_clocksource(void)
217{
218	if (!notsc) {
219		clocksource_tsc.mult = clocksource_khz2mult(tsc_khz,
220							clocksource_tsc.shift);
221		if (check_tsc_unstable())
222			clocksource_tsc.rating = 0;
223
224		clocksource_register(&clocksource_tsc);
225	}
226}
227