1#include <linux/kernel.h> 2#include <linux/sched.h> 3#include <linux/interrupt.h> 4#include <linux/init.h> 5#include <linux/clocksource.h> 6#include <linux/time.h> 7#include <linux/acpi.h> 8#include <linux/cpufreq.h> 9 10#include <asm/timex.h> 11 12static int notsc __initdata = 0; 13 14unsigned int cpu_khz; /* TSC clocks / usec, not used here */ 15EXPORT_SYMBOL(cpu_khz); 16unsigned int tsc_khz; 17EXPORT_SYMBOL(tsc_khz); 18 19static unsigned int cyc2ns_scale __read_mostly; 20 21void set_cyc2ns_scale(unsigned long khz) 22{ 23 cyc2ns_scale = (NSEC_PER_MSEC << NS_SCALE) / khz; 24} 25 26static unsigned long long cycles_2_ns(unsigned long long cyc) 27{ 28 return (cyc * cyc2ns_scale) >> NS_SCALE; 29} 30 31unsigned long long sched_clock(void) 32{ 33 unsigned long a = 0; 34 35 /* Could do CPU core sync here. Opteron can execute rdtsc speculatively, 36 * which means it is not completely exact and may not be monotonous 37 * between CPUs. But the errors should be too small to matter for 38 * scheduling purposes. 39 */ 40 41 rdtscll(a); 42 return cycles_2_ns(a); 43} 44 45static int tsc_unstable; 46 47static inline int check_tsc_unstable(void) 48{ 49 return tsc_unstable; 50} 51#ifdef CONFIG_CPU_FREQ 52 53/* Frequency scaling support. Adjust the TSC based timer when the cpu frequency 54 * changes. 55 * 56 * RED-PEN: On SMP we assume all CPUs run with the same frequency. It's 57 * not that important because current Opteron setups do not support 58 * scaling on SMP anyroads. 59 * 60 * Should fix up last_tsc too. Currently gettimeofday in the 61 * first tick after the change will be slightly wrong. 62 */ 63 64#include <linux/workqueue.h> 65 66static unsigned int cpufreq_delayed_issched = 0; 67static unsigned int cpufreq_init = 0; 68static struct work_struct cpufreq_delayed_get_work; 69 70static void handle_cpufreq_delayed_get(struct work_struct *v) 71{ 72 unsigned int cpu; 73 for_each_online_cpu(cpu) { 74 cpufreq_get(cpu); 75 } 76 cpufreq_delayed_issched = 0; 77} 78 79static unsigned int ref_freq = 0; 80static unsigned long loops_per_jiffy_ref = 0; 81 82static unsigned long tsc_khz_ref = 0; 83 84static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, 85 void *data) 86{ 87 struct cpufreq_freqs *freq = data; 88 unsigned long *lpj, dummy; 89 90 if (cpu_has(&cpu_data[freq->cpu], X86_FEATURE_CONSTANT_TSC)) 91 return 0; 92 93 lpj = &dummy; 94 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) 95#ifdef CONFIG_SMP 96 lpj = &cpu_data[freq->cpu].loops_per_jiffy; 97#else 98 lpj = &boot_cpu_data.loops_per_jiffy; 99#endif 100 101 if (!ref_freq) { 102 ref_freq = freq->old; 103 loops_per_jiffy_ref = *lpj; 104 tsc_khz_ref = tsc_khz; 105 } 106 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || 107 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) || 108 (val == CPUFREQ_RESUMECHANGE)) { 109 *lpj = 110 cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new); 111 112 tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new); 113 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) 114 mark_tsc_unstable("cpufreq changes"); 115 } 116 117 set_cyc2ns_scale(tsc_khz_ref); 118 119 return 0; 120} 121 122static struct notifier_block time_cpufreq_notifier_block = { 123 .notifier_call = time_cpufreq_notifier 124}; 125 126static int __init cpufreq_tsc(void) 127{ 128 INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get); 129 if (!cpufreq_register_notifier(&time_cpufreq_notifier_block, 130 CPUFREQ_TRANSITION_NOTIFIER)) 131 cpufreq_init = 1; 132 return 0; 133} 134 135core_initcall(cpufreq_tsc); 136 137#endif 138 139static int tsc_unstable = 0; 140 141/* 142 * Make an educated guess if the TSC is trustworthy and synchronized 143 * over all CPUs. 144 */ 145__cpuinit int unsynchronized_tsc(void) 146{ 147 if (tsc_unstable) 148 return 1; 149 150#ifdef CONFIG_SMP 151 if (apic_is_clustered_box()) 152 return 1; 153#endif 154 /* Most intel systems have synchronized TSCs except for 155 multi node systems */ 156 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 157#ifdef CONFIG_ACPI 158 /* But TSC doesn't tick in C3 so don't use it there */ 159 if (acpi_gbl_FADT.header.length > 0 && acpi_gbl_FADT.C3latency < 1000) 160 return 1; 161#endif 162 return 0; 163 } 164 165 /* Assume multi socket systems are not synchronized */ 166 return num_present_cpus() > 1; 167} 168 169int __init notsc_setup(char *s) 170{ 171 notsc = 1; 172 return 1; 173} 174 175__setup("notsc", notsc_setup); 176 177 178/* clock source code: */ 179static cycle_t read_tsc(void) 180{ 181 cycle_t ret = (cycle_t)get_cycles_sync(); 182 return ret; 183} 184 185static cycle_t __vsyscall_fn vread_tsc(void) 186{ 187 cycle_t ret = (cycle_t)get_cycles_sync(); 188 return ret; 189} 190 191static struct clocksource clocksource_tsc = { 192 .name = "tsc", 193 .rating = 300, 194 .read = read_tsc, 195 .mask = CLOCKSOURCE_MASK(64), 196 .shift = 22, 197 .flags = CLOCK_SOURCE_IS_CONTINUOUS | 198 CLOCK_SOURCE_MUST_VERIFY, 199 .vread = vread_tsc, 200}; 201 202void mark_tsc_unstable(char *reason) 203{ 204 if (!tsc_unstable) { 205 tsc_unstable = 1; 206 printk("Marking TSC unstable due to %s\n", reason); 207 /* Change only the rating, when not registered */ 208 if (clocksource_tsc.mult) 209 clocksource_change_rating(&clocksource_tsc, 0); 210 else 211 clocksource_tsc.rating = 0; 212 } 213} 214EXPORT_SYMBOL_GPL(mark_tsc_unstable); 215 216void __init init_tsc_clocksource(void) 217{ 218 if (!notsc) { 219 clocksource_tsc.mult = clocksource_khz2mult(tsc_khz, 220 clocksource_tsc.shift); 221 if (check_tsc_unstable()) 222 clocksource_tsc.rating = 0; 223 224 clocksource_register(&clocksource_tsc); 225 } 226} 227