1/* 2 * linux/kernel/time/timekeeping.c 3 * 4 * Kernel timekeeping code and accessor functions 5 * 6 * This code was moved from linux/kernel/timer.c. 7 * Please see that file for copyright and history logs. 8 * 9 */ 10 11#include <linux/module.h> 12#include <linux/interrupt.h> 13#include <linux/percpu.h> 14#include <linux/init.h> 15#include <linux/mm.h> 16#include <linux/sysdev.h> 17#include <linux/clocksource.h> 18#include <linux/jiffies.h> 19#include <linux/time.h> 20#include <linux/tick.h> 21 22 23/* 24 * This read-write spinlock protects us from races in SMP while 25 * playing with xtime and avenrun. 26 */ 27__attribute__((weak)) __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); 28 29EXPORT_SYMBOL(xtime_lock); 30 31 32/* 33 * The current time 34 * wall_to_monotonic is what we need to add to xtime (or xtime corrected 35 * for sub jiffie times) to get to monotonic time. Monotonic is pegged 36 * at zero at system boot time, so wall_to_monotonic will be negative, 37 * however, we will ALWAYS keep the tv_nsec part positive so we can use 38 * the usual normalization. 39 */ 40struct timespec xtime __attribute__ ((aligned (16))); 41struct timespec wall_to_monotonic __attribute__ ((aligned (16))); 42 43EXPORT_SYMBOL(xtime); 44 45 46static struct clocksource *clock; /* pointer to current clocksource */ 47 48 49#ifdef CONFIG_GENERIC_TIME 50/** 51 * __get_nsec_offset - Returns nanoseconds since last call to periodic_hook 52 * 53 * private function, must hold xtime_lock lock when being 54 * called. Returns the number of nanoseconds since the 55 * last call to update_wall_time() (adjusted by NTP scaling) 56 */ 57static inline s64 __get_nsec_offset(void) 58{ 59 cycle_t cycle_now, cycle_delta; 60 s64 ns_offset; 61 62 /* read clocksource: */ 63 cycle_now = clocksource_read(clock); 64 65 /* calculate the delta since the last update_wall_time: */ 66 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; 67 68 /* convert to nanoseconds: */ 69 ns_offset = cyc2ns(clock, cycle_delta); 70 71 return ns_offset; 72} 73 74/** 75 * __get_realtime_clock_ts - Returns the time of day in a timespec 76 * @ts: pointer to the timespec to be set 77 * 78 * Returns the time of day in a timespec. Used by 79 * do_gettimeofday() and get_realtime_clock_ts(). 80 */ 81static inline void __get_realtime_clock_ts(struct timespec *ts) 82{ 83 unsigned long seq; 84 s64 nsecs; 85 86 do { 87 seq = read_seqbegin(&xtime_lock); 88 89 *ts = xtime; 90 nsecs = __get_nsec_offset(); 91 92 } while (read_seqretry(&xtime_lock, seq)); 93 94 timespec_add_ns(ts, nsecs); 95} 96 97/** 98 * getnstimeofday - Returns the time of day in a timespec 99 * @ts: pointer to the timespec to be set 100 * 101 * Returns the time of day in a timespec. 102 */ 103void getnstimeofday(struct timespec *ts) 104{ 105 __get_realtime_clock_ts(ts); 106} 107 108EXPORT_SYMBOL(getnstimeofday); 109 110/** 111 * do_gettimeofday - Returns the time of day in a timeval 112 * @tv: pointer to the timeval to be set 113 * 114 * NOTE: Users should be converted to using get_realtime_clock_ts() 115 */ 116void do_gettimeofday(struct timeval *tv) 117{ 118 struct timespec now; 119 120 __get_realtime_clock_ts(&now); 121 tv->tv_sec = now.tv_sec; 122 tv->tv_usec = now.tv_nsec/1000; 123} 124 125EXPORT_SYMBOL(do_gettimeofday); 126/** 127 * do_settimeofday - Sets the time of day 128 * @tv: pointer to the timespec variable containing the new time 129 * 130 * Sets the time of day to the new time and update NTP and notify hrtimers 131 */ 132int do_settimeofday(struct timespec *tv) 133{ 134 unsigned long flags; 135 time_t wtm_sec, sec = tv->tv_sec; 136 long wtm_nsec, nsec = tv->tv_nsec; 137 138 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) 139 return -EINVAL; 140 141 write_seqlock_irqsave(&xtime_lock, flags); 142 143 nsec -= __get_nsec_offset(); 144 145 wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec); 146 wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec); 147 148 set_normalized_timespec(&xtime, sec, nsec); 149 set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec); 150 151 clock->error = 0; 152 ntp_clear(); 153 154 update_vsyscall(&xtime, clock); 155 156 write_sequnlock_irqrestore(&xtime_lock, flags); 157 158 /* signal hrtimers about time change */ 159 clock_was_set(); 160 161 return 0; 162} 163 164EXPORT_SYMBOL(do_settimeofday); 165 166/** 167 * change_clocksource - Swaps clocksources if a new one is available 168 * 169 * Accumulates current time interval and initializes new clocksource 170 */ 171static void change_clocksource(void) 172{ 173 struct clocksource *new; 174 cycle_t now; 175 u64 nsec; 176 177 new = clocksource_get_next(); 178 179 if (clock == new) 180 return; 181 182 now = clocksource_read(new); 183 nsec = __get_nsec_offset(); 184 timespec_add_ns(&xtime, nsec); 185 186 clock = new; 187 clock->cycle_last = now; 188 189 clock->error = 0; 190 clock->xtime_nsec = 0; 191 clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); 192 193 tick_clock_notify(); 194 195 printk(KERN_INFO "Time: %s clocksource has been installed.\n", 196 clock->name); 197} 198#else 199static inline void change_clocksource(void) { } 200#endif 201 202/** 203 * timekeeping_is_continuous - check to see if timekeeping is free running 204 */ 205int timekeeping_is_continuous(void) 206{ 207 unsigned long seq; 208 int ret; 209 210 do { 211 seq = read_seqbegin(&xtime_lock); 212 213 ret = clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; 214 215 } while (read_seqretry(&xtime_lock, seq)); 216 217 return ret; 218} 219 220unsigned long __attribute__((weak)) read_persistent_clock(void) 221{ 222 return 0; 223} 224 225/* 226 * timekeeping_init - Initializes the clocksource and common timekeeping values 227 */ 228void __init timekeeping_init(void) 229{ 230 unsigned long flags; 231 unsigned long sec = read_persistent_clock(); 232 233 write_seqlock_irqsave(&xtime_lock, flags); 234 235 ntp_clear(); 236 237 clock = clocksource_get_next(); 238 clocksource_calculate_interval(clock, NTP_INTERVAL_LENGTH); 239 clock->cycle_last = clocksource_read(clock); 240 241 xtime.tv_sec = sec; 242 xtime.tv_nsec = 0; 243 set_normalized_timespec(&wall_to_monotonic, 244 -xtime.tv_sec, -xtime.tv_nsec); 245 246 write_sequnlock_irqrestore(&xtime_lock, flags); 247} 248 249/* flag for if timekeeping is suspended */ 250static int timekeeping_suspended; 251/* time in seconds when suspend began */ 252static unsigned long timekeeping_suspend_time; 253 254/** 255 * timekeeping_resume - Resumes the generic timekeeping subsystem. 256 * @dev: unused 257 * 258 * This is for the generic clocksource timekeeping. 259 * xtime/wall_to_monotonic/jiffies/etc are 260 * still managed by arch specific suspend/resume code. 261 */ 262static int timekeeping_resume(struct sys_device *dev) 263{ 264 unsigned long flags; 265 unsigned long now = read_persistent_clock(); 266 267 clocksource_resume(); 268 269 write_seqlock_irqsave(&xtime_lock, flags); 270 271 if (now && (now > timekeeping_suspend_time)) { 272 unsigned long sleep_length = now - timekeeping_suspend_time; 273 274 xtime.tv_sec += sleep_length; 275 wall_to_monotonic.tv_sec -= sleep_length; 276 } 277 /* re-base the last cycle value */ 278 clock->cycle_last = clocksource_read(clock); 279 clock->error = 0; 280 timekeeping_suspended = 0; 281 write_sequnlock_irqrestore(&xtime_lock, flags); 282 283 touch_softlockup_watchdog(); 284 285 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL); 286 287 /* Resume hrtimers */ 288 hres_timers_resume(); 289 290 return 0; 291} 292 293static int timekeeping_suspend(struct sys_device *dev, pm_message_t state) 294{ 295 unsigned long flags; 296 297 write_seqlock_irqsave(&xtime_lock, flags); 298 timekeeping_suspended = 1; 299 timekeeping_suspend_time = read_persistent_clock(); 300 write_sequnlock_irqrestore(&xtime_lock, flags); 301 302 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); 303 304 return 0; 305} 306 307/* sysfs resume/suspend bits for timekeeping */ 308static struct sysdev_class timekeeping_sysclass = { 309 .resume = timekeeping_resume, 310 .suspend = timekeeping_suspend, 311 set_kset_name("timekeeping"), 312}; 313 314static struct sys_device device_timer = { 315 .id = 0, 316 .cls = &timekeeping_sysclass, 317}; 318 319static int __init timekeeping_init_device(void) 320{ 321 int error = sysdev_class_register(&timekeeping_sysclass); 322 if (!error) 323 error = sysdev_register(&device_timer); 324 return error; 325} 326 327device_initcall(timekeeping_init_device); 328 329/* 330 * If the error is already larger, we look ahead even further 331 * to compensate for late or lost adjustments. 332 */ 333static __always_inline int clocksource_bigadjust(s64 error, s64 *interval, 334 s64 *offset) 335{ 336 s64 tick_error, i; 337 u32 look_ahead, adj; 338 s32 error2, mult; 339 340 /* 341 * Use the current error value to determine how much to look ahead. 342 * The larger the error the slower we adjust for it to avoid problems 343 * with losing too many ticks, otherwise we would overadjust and 344 * produce an even larger error. The smaller the adjustment the 345 * faster we try to adjust for it, as lost ticks can do less harm 346 * here. This is tuned so that an error of about 1 msec is adusted 347 * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks). 348 */ 349 error2 = clock->error >> (TICK_LENGTH_SHIFT + 22 - 2 * SHIFT_HZ); 350 error2 = abs(error2); 351 for (look_ahead = 0; error2 > 0; look_ahead++) 352 error2 >>= 2; 353 354 /* 355 * Now calculate the error in (1 << look_ahead) ticks, but first 356 * remove the single look ahead already included in the error. 357 */ 358 tick_error = current_tick_length() >> 359 (TICK_LENGTH_SHIFT - clock->shift + 1); 360 tick_error -= clock->xtime_interval >> 1; 361 error = ((error - tick_error) >> look_ahead) + tick_error; 362 363 /* Finally calculate the adjustment shift value. */ 364 i = *interval; 365 mult = 1; 366 if (error < 0) { 367 error = -error; 368 *interval = -*interval; 369 *offset = -*offset; 370 mult = -1; 371 } 372 for (adj = 0; error > i; adj++) 373 error >>= 1; 374 375 *interval <<= adj; 376 *offset <<= adj; 377 return mult << adj; 378} 379 380/* 381 * Adjust the multiplier to reduce the error value, 382 * this is optimized for the most common adjustments of -1,0,1, 383 * for other values we can do a bit more work. 384 */ 385static void clocksource_adjust(struct clocksource *clock, s64 offset) 386{ 387 s64 error, interval = clock->cycle_interval; 388 int adj; 389 390 error = clock->error >> (TICK_LENGTH_SHIFT - clock->shift - 1); 391 if (error > interval) { 392 error >>= 2; 393 if (likely(error <= interval)) 394 adj = 1; 395 else 396 adj = clocksource_bigadjust(error, &interval, &offset); 397 } else if (error < -interval) { 398 error >>= 2; 399 if (likely(error >= -interval)) { 400 adj = -1; 401 interval = -interval; 402 offset = -offset; 403 } else 404 adj = clocksource_bigadjust(error, &interval, &offset); 405 } else 406 return; 407 408 clock->mult += adj; 409 clock->xtime_interval += interval; 410 clock->xtime_nsec -= offset; 411 clock->error -= (interval - offset) << 412 (TICK_LENGTH_SHIFT - clock->shift); 413} 414 415/** 416 * update_wall_time - Uses the current clocksource to increment the wall time 417 * 418 * Called from the timer interrupt, must hold a write on xtime_lock. 419 */ 420void update_wall_time(void) 421{ 422 cycle_t offset; 423 424 /* Make sure we're fully resumed: */ 425 if (unlikely(timekeeping_suspended)) 426 return; 427 428#ifdef CONFIG_GENERIC_TIME 429 offset = (clocksource_read(clock) - clock->cycle_last) & clock->mask; 430#else 431 offset = clock->cycle_interval; 432#endif 433 clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift; 434 435 /* normally this loop will run just once, however in the 436 * case of lost or late ticks, it will accumulate correctly. 437 */ 438 while (offset >= clock->cycle_interval) { 439 /* accumulate one interval */ 440 clock->xtime_nsec += clock->xtime_interval; 441 clock->cycle_last += clock->cycle_interval; 442 offset -= clock->cycle_interval; 443 444 if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) { 445 clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift; 446 xtime.tv_sec++; 447 second_overflow(); 448 } 449 450 /* interpolator bits */ 451 time_interpolator_update(clock->xtime_interval 452 >> clock->shift); 453 454 /* accumulate error between NTP and clock interval */ 455 clock->error += current_tick_length(); 456 clock->error -= clock->xtime_interval << (TICK_LENGTH_SHIFT - clock->shift); 457 } 458 459 /* correct the clock when NTP error is too big */ 460 clocksource_adjust(clock, offset); 461 462 /* store full nanoseconds into xtime */ 463 xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift; 464 clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift; 465 466 /* check to see if there is a new clocksource to use */ 467 change_clocksource(); 468 update_vsyscall(&xtime, clock); 469} 470