1/* 2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31 32/* 33 * File: i386/rtclock.c 34 * Purpose: Routines for handling the machine dependent 35 * real-time clock. Historically, this clock is 36 * generated by the Intel 8254 Programmable Interval 37 * Timer, but local apic timers are now used for 38 * this purpose with the master time reference being 39 * the cpu clock counted by the timestamp MSR. 40 */ 41 42 43#include <mach/mach_types.h> 44 45#include <kern/cpu_data.h> 46#include <kern/cpu_number.h> 47#include <kern/clock.h> 48#include <kern/host_notify.h> 49#include <kern/macro_help.h> 50#include <kern/misc_protos.h> 51#include <kern/spl.h> 52#include <kern/assert.h> 53#include <kern/timer_queue.h> 54#include <mach/vm_prot.h> 55#include <vm/pmap.h> 56#include <vm/vm_kern.h> /* for kernel_map */ 57#include <architecture/i386/pio.h> 58#include <i386/machine_cpu.h> 59#include <i386/cpuid.h> 60#include <i386/cpu_threads.h> 61#include <i386/mp.h> 62#include <i386/machine_routines.h> 63#include <i386/pal_routines.h> 64#include <i386/proc_reg.h> 65#include <i386/misc_protos.h> 66#include <pexpert/pexpert.h> 67#include <machine/limits.h> 68#include <machine/commpage.h> 69#include <sys/kdebug.h> 70#include <i386/tsc.h> 71#include <i386/rtclock_protos.h> 72#define UI_CPUFREQ_ROUNDING_FACTOR 10000000 73 74int rtclock_init(void); 75 76uint64_t tsc_rebase_abs_time = 0; 77 78static void rtc_set_timescale(uint64_t cycles); 79static uint64_t rtc_export_speed(uint64_t cycles); 80 81void 82rtc_timer_start(void) 83{ 84 /* 85 * Force a complete re-evaluation of timer deadlines. 86 */ 87 x86_lcpu()->rtcDeadline = EndOfAllTime; 88 timer_resync_deadlines(); 89} 90 91static inline uint32_t 92_absolutetime_to_microtime(uint64_t abstime, clock_sec_t *secs, clock_usec_t *microsecs) 93{ 94 uint32_t remain; 95 *secs = abstime / (uint64_t)NSEC_PER_SEC; 96 remain = (uint32_t)(abstime % (uint64_t)NSEC_PER_SEC); 97 *microsecs = remain / NSEC_PER_USEC; 98 return remain; 99} 100 101static inline void 102_absolutetime_to_nanotime(uint64_t abstime, clock_sec_t *secs, clock_usec_t *nanosecs) 103{ 104 *secs = abstime / (uint64_t)NSEC_PER_SEC; 105 *nanosecs = (clock_usec_t)(abstime % (uint64_t)NSEC_PER_SEC); 106} 107 108/* 109 * Nanotime/mach_absolutime_time 110 * ----------------------------- 111 * The timestamp counter (TSC) - which counts cpu clock cycles and can be read 112 * efficiently by the kernel and in userspace - is the reference for all timing. 113 * The cpu clock rate is platform-dependent and may stop or be reset when the 114 * processor is napped/slept. As a result, nanotime is the software abstraction 115 * used to maintain a monotonic clock, adjusted from an outside reference as needed. 116 * 117 * The kernel maintains nanotime information recording: 118 * - the ratio of tsc to nanoseconds 119 * with this ratio expressed as a 32-bit scale and shift 120 * (power of 2 divider); 121 * - { tsc_base, ns_base } pair of corresponding timestamps. 122 * 123 * The tuple {tsc_base, ns_base, scale, shift} is exported in the commpage 124 * for the userspace nanotime routine to read. 125 * 126 * All of the routines which update the nanotime data are non-reentrant. This must 127 * be guaranteed by the caller. 128 */ 129static inline void 130rtc_nanotime_set_commpage(pal_rtc_nanotime_t *rntp) 131{ 132 commpage_set_nanotime(rntp->tsc_base, rntp->ns_base, rntp->scale, rntp->shift); 133} 134 135/* 136 * rtc_nanotime_init: 137 * 138 * Intialize the nanotime info from the base time. 139 */ 140static inline void 141_rtc_nanotime_init(pal_rtc_nanotime_t *rntp, uint64_t base) 142{ 143 uint64_t tsc = rdtsc64(); 144 145 _pal_rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp); 146} 147 148static void 149rtc_nanotime_init(uint64_t base) 150{ 151 _rtc_nanotime_init(&pal_rtc_nanotime_info, base); 152 rtc_nanotime_set_commpage(&pal_rtc_nanotime_info); 153} 154 155/* 156 * rtc_nanotime_init_commpage: 157 * 158 * Call back from the commpage initialization to 159 * cause the commpage data to be filled in once the 160 * commpages have been created. 161 */ 162void 163rtc_nanotime_init_commpage(void) 164{ 165 spl_t s = splclock(); 166 167 rtc_nanotime_set_commpage(&pal_rtc_nanotime_info); 168 splx(s); 169} 170 171/* 172 * rtc_nanotime_read: 173 * 174 * Returns the current nanotime value, accessable from any 175 * context. 176 */ 177static inline uint64_t 178rtc_nanotime_read(void) 179{ 180 return _rtc_nanotime_read(&pal_rtc_nanotime_info); 181} 182 183/* 184 * rtc_clock_napped: 185 * 186 * Invoked from power management when we exit from a low C-State (>= C4) 187 * and the TSC has stopped counting. The nanotime data is updated according 188 * to the provided value which represents the new value for nanotime. 189 */ 190void 191rtc_clock_napped(uint64_t base, uint64_t tsc_base) 192{ 193 pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; 194 uint64_t oldnsecs; 195 uint64_t newnsecs; 196 uint64_t tsc; 197 198 assert(!ml_get_interrupts_enabled()); 199 tsc = rdtsc64(); 200 oldnsecs = rntp->ns_base + _rtc_tsc_to_nanoseconds(tsc - rntp->tsc_base, rntp); 201 newnsecs = base + _rtc_tsc_to_nanoseconds(tsc - tsc_base, rntp); 202 203 /* 204 * Only update the base values if time using the new base values 205 * is later than the time using the old base values. 206 */ 207 if (oldnsecs < newnsecs) { 208 _pal_rtc_nanotime_store(tsc_base, base, rntp->scale, rntp->shift, rntp); 209 rtc_nanotime_set_commpage(rntp); 210 } 211} 212 213/* 214 * Invoked from power management to correct the SFLM TSC entry drift problem: 215 * a small delta is added to the tsc_base. This is equivalent to nudgin time 216 * backwards. We require this to be on the order of a TSC quantum which won't 217 * cause callers of mach_absolute_time() to see time going backwards! 218 */ 219void 220rtc_clock_adjust(uint64_t tsc_base_delta) 221{ 222 pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; 223 224 assert(!ml_get_interrupts_enabled()); 225 assert(tsc_base_delta < 100ULL); /* i.e. it's small */ 226 _rtc_nanotime_adjust(tsc_base_delta, rntp); 227 rtc_nanotime_set_commpage(rntp); 228} 229 230void 231rtc_clock_stepping(__unused uint32_t new_frequency, 232 __unused uint32_t old_frequency) 233{ 234 panic("rtc_clock_stepping unsupported"); 235} 236 237void 238rtc_clock_stepped(__unused uint32_t new_frequency, 239 __unused uint32_t old_frequency) 240{ 241 panic("rtc_clock_stepped unsupported"); 242} 243 244/* 245 * rtc_sleep_wakeup: 246 * 247 * Invoked from power management when we have awoken from a sleep (S3) 248 * and the TSC has been reset, or from Deep Idle (S0) sleep when the TSC 249 * has progressed. The nanotime data is updated based on the passed-in value. 250 * 251 * The caller must guarantee non-reentrancy. 252 */ 253void 254rtc_sleep_wakeup( 255 uint64_t base) 256{ 257 /* Set fixed configuration for lapic timers */ 258 rtc_timer->rtc_config(); 259 260 /* 261 * Reset nanotime. 262 * The timestamp counter will have been reset 263 * but nanotime (uptime) marches onward. 264 */ 265 rtc_nanotime_init(base); 266} 267 268/* 269 * rtclock_early_init() is called very early at boot to 270 * establish mach_absolute_time() and set it to zero. 271 */ 272void 273rtclock_early_init(void) 274{ 275 assert(tscFreq); 276 rtc_set_timescale(tscFreq); 277} 278 279/* 280 * Initialize the real-time clock device. 281 * In addition, various variables used to support the clock are initialized. 282 */ 283int 284rtclock_init(void) 285{ 286 uint64_t cycles; 287 288 assert(!ml_get_interrupts_enabled()); 289 290 if (cpu_number() == master_cpu) { 291 292 assert(tscFreq); 293 294 /* 295 * Adjust and set the exported cpu speed. 296 */ 297 cycles = rtc_export_speed(tscFreq); 298 299 /* 300 * Set min/max to actual. 301 * ACPI may update these later if speed-stepping is detected. 302 */ 303 gPEClockFrequencyInfo.cpu_frequency_min_hz = cycles; 304 gPEClockFrequencyInfo.cpu_frequency_max_hz = cycles; 305 306 rtc_timer_init(); 307 clock_timebase_init(); 308 ml_init_lock_timeout(); 309 ml_init_delay_spin_threshold(10); 310 } 311 312 /* Set fixed configuration for lapic timers */ 313 rtc_timer->rtc_config(); 314 rtc_timer_start(); 315 316 return (1); 317} 318 319// utility routine 320// Code to calculate how many processor cycles are in a second... 321 322static void 323rtc_set_timescale(uint64_t cycles) 324{ 325 pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; 326 uint32_t shift = 0; 327 328 /* the "scale" factor will overflow unless cycles>SLOW_TSC_THRESHOLD */ 329 330 while ( cycles <= SLOW_TSC_THRESHOLD) { 331 shift++; 332 cycles <<= 1; 333 } 334 335 rntp->scale = (uint32_t)(((uint64_t)NSEC_PER_SEC << 32) / cycles); 336 337 rntp->shift = shift; 338 339 /* 340 * On some platforms, the TSC is not reset at warm boot. But the 341 * rebase time must be relative to the current boot so we can't use 342 * mach_absolute_time(). Instead, we convert the TSC delta since boot 343 * to nanoseconds. 344 */ 345 if (tsc_rebase_abs_time == 0) 346 tsc_rebase_abs_time = _rtc_tsc_to_nanoseconds( 347 rdtsc64() - tsc_at_boot, rntp); 348 349 rtc_nanotime_init(0); 350} 351 352static uint64_t 353rtc_export_speed(uint64_t cyc_per_sec) 354{ 355 pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; 356 uint64_t cycles; 357 358 if (rntp->shift != 0 ) 359 printf("Slow TSC, rtc_nanotime.shift == %d\n", rntp->shift); 360 361 /* Round: */ 362 cycles = ((cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR/2)) 363 / UI_CPUFREQ_ROUNDING_FACTOR) 364 * UI_CPUFREQ_ROUNDING_FACTOR; 365 366 /* 367 * Set current measured speed. 368 */ 369 if (cycles >= 0x100000000ULL) { 370 gPEClockFrequencyInfo.cpu_clock_rate_hz = 0xFFFFFFFFUL; 371 } else { 372 gPEClockFrequencyInfo.cpu_clock_rate_hz = (unsigned long)cycles; 373 } 374 gPEClockFrequencyInfo.cpu_frequency_hz = cycles; 375 376 kprintf("[RTCLOCK] frequency %llu (%llu)\n", cycles, cyc_per_sec); 377 return(cycles); 378} 379 380void 381clock_get_system_microtime( 382 clock_sec_t *secs, 383 clock_usec_t *microsecs) 384{ 385 uint64_t now = rtc_nanotime_read(); 386 387 _absolutetime_to_microtime(now, secs, microsecs); 388} 389 390void 391clock_get_system_nanotime( 392 clock_sec_t *secs, 393 clock_nsec_t *nanosecs) 394{ 395 uint64_t now = rtc_nanotime_read(); 396 397 _absolutetime_to_nanotime(now, secs, nanosecs); 398} 399 400void 401clock_gettimeofday_set_commpage( 402 uint64_t abstime, 403 uint64_t epoch, 404 uint64_t offset, 405 clock_sec_t *secs, 406 clock_usec_t *microsecs) 407{ 408 uint64_t now = abstime + offset; 409 uint32_t remain; 410 411 remain = _absolutetime_to_microtime(now, secs, microsecs); 412 413 *secs += (clock_sec_t)epoch; 414 415 commpage_set_timestamp(abstime - remain, *secs); 416} 417 418void 419clock_timebase_info( 420 mach_timebase_info_t info) 421{ 422 info->numer = info->denom = 1; 423} 424 425/* 426 * Real-time clock device interrupt. 427 */ 428void 429rtclock_intr( 430 x86_saved_state_t *tregs) 431{ 432 uint64_t rip; 433 boolean_t user_mode = FALSE; 434 435 assert(get_preemption_level() > 0); 436 assert(!ml_get_interrupts_enabled()); 437 438 if (is_saved_state64(tregs) == TRUE) { 439 x86_saved_state64_t *regs; 440 441 regs = saved_state64(tregs); 442 443 if (regs->isf.cs & 0x03) 444 user_mode = TRUE; 445 rip = regs->isf.rip; 446 } else { 447 x86_saved_state32_t *regs; 448 449 regs = saved_state32(tregs); 450 451 if (regs->cs & 0x03) 452 user_mode = TRUE; 453 rip = regs->eip; 454 } 455 456 /* call the generic etimer */ 457 timer_intr(user_mode, rip); 458} 459 460 461/* 462 * Request timer pop from the hardware 463 */ 464 465uint64_t 466setPop(uint64_t time) 467{ 468 uint64_t now; 469 uint64_t pop; 470 471 /* 0 and EndOfAllTime are special-cases for "clear the timer" */ 472 if (time == 0 || time == EndOfAllTime ) { 473 time = EndOfAllTime; 474 now = 0; 475 pop = rtc_timer->rtc_set(0, 0); 476 } else { 477 now = rtc_nanotime_read(); /* The time in nanoseconds */ 478 pop = rtc_timer->rtc_set(time, now); 479 } 480 481 /* Record requested and actual deadlines set */ 482 x86_lcpu()->rtcDeadline = time; 483 x86_lcpu()->rtcPop = pop; 484 485 return pop - now; 486} 487 488uint64_t 489mach_absolute_time(void) 490{ 491 return rtc_nanotime_read(); 492} 493 494void 495clock_interval_to_absolutetime_interval( 496 uint32_t interval, 497 uint32_t scale_factor, 498 uint64_t *result) 499{ 500 *result = (uint64_t)interval * scale_factor; 501} 502 503void 504absolutetime_to_microtime( 505 uint64_t abstime, 506 clock_sec_t *secs, 507 clock_usec_t *microsecs) 508{ 509 _absolutetime_to_microtime(abstime, secs, microsecs); 510} 511 512void 513nanotime_to_absolutetime( 514 clock_sec_t secs, 515 clock_nsec_t nanosecs, 516 uint64_t *result) 517{ 518 *result = ((uint64_t)secs * NSEC_PER_SEC) + nanosecs; 519} 520 521void 522absolutetime_to_nanoseconds( 523 uint64_t abstime, 524 uint64_t *result) 525{ 526 *result = abstime; 527} 528 529void 530nanoseconds_to_absolutetime( 531 uint64_t nanoseconds, 532 uint64_t *result) 533{ 534 *result = nanoseconds; 535} 536 537void 538machine_delay_until( 539 uint64_t interval, 540 uint64_t deadline) 541{ 542 (void)interval; 543 while (mach_absolute_time() < deadline) { 544 cpu_pause(); 545 } 546} 547