1/* 2 * This code largely moved from arch/i386/kernel/timer/timer_tsc.c 3 * which was originally moved from arch/i386/kernel/time.c. 4 * See comments there for proper credits. 5 */ 6 7#include <linux/clocksource.h> 8#include <linux/workqueue.h> 9#include <linux/cpufreq.h> 10#include <linux/jiffies.h> 11#include <linux/init.h> 12#include <linux/dmi.h> 13 14#include <asm/delay.h> 15#include <asm/tsc.h> 16#include <asm/io.h> 17#include <asm/timer.h> 18 19#include "mach_timer.h" 20 21static int tsc_enabled; 22 23/* 24 * On some systems the TSC frequency does not 25 * change with the cpu frequency. So we need 26 * an extra value to store the TSC freq 27 */ 28unsigned int tsc_khz; 29 30int tsc_disable; 31 32#ifdef CONFIG_X86_TSC 33static int __init tsc_setup(char *str) 34{ 35 printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, " 36 "cannot disable TSC.\n"); 37 return 1; 38} 39#else 40/* 41 * disable flag for tsc. Takes effect by clearing the TSC cpu flag 42 * in cpu/common.c 43 */ 44static int __init tsc_setup(char *str) 45{ 46 tsc_disable = 1; 47 48 return 1; 49} 50#endif 51 52__setup("notsc", tsc_setup); 53 54/* 55 * code to mark and check if the TSC is unstable 56 * due to cpufreq or due to unsynced TSCs 57 */ 58static int tsc_unstable; 59 60static inline int check_tsc_unstable(void) 61{ 62 return tsc_unstable; 63} 64 65/* Accellerators for sched_clock() 66 * convert from cycles(64bits) => nanoseconds (64bits) 67 * basic equation: 68 * ns = cycles / (freq / ns_per_sec) 69 * ns = cycles * (ns_per_sec / freq) 70 * ns = cycles * (10^9 / (cpu_khz * 10^3)) 71 * ns = cycles * (10^6 / cpu_khz) 72 * 73 * Then we use scaling math (suggested by george@mvista.com) to get: 74 * ns = cycles * (10^6 * SC / cpu_khz) / SC 75 * ns = cycles * cyc2ns_scale / SC 76 * 77 * And since SC is a constant power of two, we can convert the div 78 * into a shift. 79 * 80 * We can use khz divisor instead of mhz to keep a better percision, since 81 * cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits. 82 * (mathieu.desnoyers@polymtl.ca) 83 * 84 * -johnstul@us.ibm.com "math is hard, lets go shopping!" 85 */ 86static unsigned long cyc2ns_scale __read_mostly; 87 88#define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */ 89 90static inline void set_cyc2ns_scale(unsigned long cpu_khz) 91{ 92 cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz; 93} 94 95static inline unsigned long long cycles_2_ns(unsigned long long cyc) 96{ 97 return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR; 98} 99 100/* 101 * Scheduler clock - returns current time in nanosec units. 102 */ 103unsigned long long sched_clock(void) 104{ 105 unsigned long long this_offset; 106 107 /* 108 * Fall back to jiffies if there's no TSC available: 109 */ 110 if (unlikely(!tsc_enabled)) 111 /* No locking but a rare wrong value is not a big deal: */ 112 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ); 113 114 /* read the Time Stamp Counter: */ 115 get_scheduled_cycles(this_offset); 116 117 /* return the value in ns */ 118 return cycles_2_ns(this_offset); 119} 120 121unsigned long native_calculate_cpu_khz(void) 122{ 123 unsigned long long start, end; 124 unsigned long count; 125 u64 delta64; 126 int i; 127 unsigned long flags; 128 129 local_irq_save(flags); 130 131 /* run 3 times to ensure the cache is warm */ 132 for (i = 0; i < 3; i++) { 133 mach_prepare_counter(); 134 rdtscll(start); 135 mach_countup(&count); 136 rdtscll(end); 137 } 138 /* 139 * Error: ECTCNEVERSET 140 * The CTC wasn't reliable: we got a hit on the very first read, 141 * or the CPU was so fast/slow that the quotient wouldn't fit in 142 * 32 bits.. 143 */ 144 if (count <= 1) 145 goto err; 146 147 delta64 = end - start; 148 149 /* cpu freq too fast: */ 150 if (delta64 > (1ULL<<32)) 151 goto err; 152 153 /* cpu freq too slow: */ 154 if (delta64 <= CALIBRATE_TIME_MSEC) 155 goto err; 156 157 delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */ 158 do_div(delta64,CALIBRATE_TIME_MSEC); 159 160 local_irq_restore(flags); 161 return (unsigned long)delta64; 162err: 163 local_irq_restore(flags); 164 return 0; 165} 166 167int recalibrate_cpu_khz(void) 168{ 169#ifndef CONFIG_SMP 170 unsigned long cpu_khz_old = cpu_khz; 171 172 if (cpu_has_tsc) { 173 cpu_khz = calculate_cpu_khz(); 174 tsc_khz = cpu_khz; 175 cpu_data[0].loops_per_jiffy = 176 cpufreq_scale(cpu_data[0].loops_per_jiffy, 177 cpu_khz_old, cpu_khz); 178 return 0; 179 } else 180 return -ENODEV; 181#else 182 return -ENODEV; 183#endif 184} 185 186EXPORT_SYMBOL(recalibrate_cpu_khz); 187 188#ifdef CONFIG_CPU_FREQ 189 190/* 191 * if the CPU frequency is scaled, TSC-based delays will need a different 192 * loops_per_jiffy value to function properly. 193 */ 194static unsigned int ref_freq = 0; 195static unsigned long loops_per_jiffy_ref = 0; 196static unsigned long cpu_khz_ref = 0; 197 198static int 199time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data) 200{ 201 struct cpufreq_freqs *freq = data; 202 203 if (!ref_freq) { 204 if (!freq->old){ 205 ref_freq = freq->new; 206 return 0; 207 } 208 ref_freq = freq->old; 209 loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy; 210 cpu_khz_ref = cpu_khz; 211 } 212 213 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || 214 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) || 215 (val == CPUFREQ_RESUMECHANGE)) { 216 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) 217 cpu_data[freq->cpu].loops_per_jiffy = 218 cpufreq_scale(loops_per_jiffy_ref, 219 ref_freq, freq->new); 220 221 if (cpu_khz) { 222 223 if (num_online_cpus() == 1) 224 cpu_khz = cpufreq_scale(cpu_khz_ref, 225 ref_freq, freq->new); 226 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) { 227 tsc_khz = cpu_khz; 228 set_cyc2ns_scale(cpu_khz); 229 /* 230 * TSC based sched_clock turns 231 * to junk w/ cpufreq 232 */ 233 mark_tsc_unstable("cpufreq changes"); 234 } 235 } 236 } 237 238 return 0; 239} 240 241static struct notifier_block time_cpufreq_notifier_block = { 242 .notifier_call = time_cpufreq_notifier 243}; 244 245static int __init cpufreq_tsc(void) 246{ 247 return cpufreq_register_notifier(&time_cpufreq_notifier_block, 248 CPUFREQ_TRANSITION_NOTIFIER); 249} 250core_initcall(cpufreq_tsc); 251 252#endif 253 254/* clock source code */ 255 256static unsigned long current_tsc_khz = 0; 257 258static cycle_t read_tsc(void) 259{ 260 cycle_t ret; 261 262 rdtscll(ret); 263 264 return ret; 265} 266 267static struct clocksource clocksource_tsc = { 268 .name = "tsc", 269 .rating = 300, 270 .read = read_tsc, 271 .mask = CLOCKSOURCE_MASK(64), 272 .mult = 0, /* to be set */ 273 .shift = 22, 274 .flags = CLOCK_SOURCE_IS_CONTINUOUS | 275 CLOCK_SOURCE_MUST_VERIFY, 276}; 277 278void mark_tsc_unstable(char *reason) 279{ 280 if (!tsc_unstable) { 281 tsc_unstable = 1; 282 tsc_enabled = 0; 283 printk("Marking TSC unstable due to: %s.\n", reason); 284 /* Can be called before registration */ 285 if (clocksource_tsc.mult) 286 clocksource_change_rating(&clocksource_tsc, 0); 287 else 288 clocksource_tsc.rating = 0; 289 } 290} 291EXPORT_SYMBOL_GPL(mark_tsc_unstable); 292 293static int __init dmi_mark_tsc_unstable(struct dmi_system_id *d) 294{ 295 printk(KERN_NOTICE "%s detected: marking TSC unstable.\n", 296 d->ident); 297 tsc_unstable = 1; 298 return 0; 299} 300 301/* List of systems that have known TSC problems */ 302static struct dmi_system_id __initdata bad_tsc_dmi_table[] = { 303 { 304 .callback = dmi_mark_tsc_unstable, 305 .ident = "IBM Thinkpad 380XD", 306 .matches = { 307 DMI_MATCH(DMI_BOARD_VENDOR, "IBM"), 308 DMI_MATCH(DMI_BOARD_NAME, "2635FA0"), 309 }, 310 }, 311 {} 312}; 313 314/* 315 * Make an educated guess if the TSC is trustworthy and synchronized 316 * over all CPUs. 317 */ 318__cpuinit int unsynchronized_tsc(void) 319{ 320 if (!cpu_has_tsc || tsc_unstable) 321 return 1; 322 /* 323 * Intel systems are normally all synchronized. 324 * Exceptions must mark TSC as unstable: 325 */ 326 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 327 /* assume multi socket systems are not synchronized: */ 328 if (num_possible_cpus() > 1) 329 tsc_unstable = 1; 330 } 331 return tsc_unstable; 332} 333 334/* 335 * Geode_LX - the OLPC CPU has a possibly a very reliable TSC 336 */ 337#ifdef CONFIG_MGEODE_LX 338/* RTSC counts during suspend */ 339#define RTSC_SUSP 0x100 340 341static void __init check_geode_tsc_reliable(void) 342{ 343 unsigned long val; 344 345 rdmsrl(MSR_GEODE_BUSCONT_CONF0, val); 346 if ((val & RTSC_SUSP)) 347 clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY; 348} 349#else 350static inline void check_geode_tsc_reliable(void) { } 351#endif 352 353 354void __init tsc_init(void) 355{ 356 if (!cpu_has_tsc || tsc_disable) 357 goto out_no_tsc; 358 359 cpu_khz = calculate_cpu_khz(); 360 tsc_khz = cpu_khz; 361 362 if (!cpu_khz) 363 goto out_no_tsc; 364 365 printk("Detected %lu.%03lu MHz processor.\n", 366 (unsigned long)cpu_khz / 1000, 367 (unsigned long)cpu_khz % 1000); 368 369 set_cyc2ns_scale(cpu_khz); 370 use_tsc_delay(); 371 372 /* Check and install the TSC clocksource */ 373 dmi_check_system(bad_tsc_dmi_table); 374 375 unsynchronized_tsc(); 376 check_geode_tsc_reliable(); 377 current_tsc_khz = tsc_khz; 378 clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz, 379 clocksource_tsc.shift); 380 /* lower the rating if we already know its unstable: */ 381 if (check_tsc_unstable()) { 382 clocksource_tsc.rating = 0; 383 clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS; 384 } else 385 tsc_enabled = 1; 386 387 clocksource_register(&clocksource_tsc); 388 389 return; 390 391out_no_tsc: 392 /* 393 * Set the tsc_disable flag if there's no TSC support, this 394 * makes it a fast flag for the kernel to see whether it 395 * should be using the TSC. 396 */ 397 tsc_disable = 1; 398} 399