1/* 2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31 32/* 33 * File: i386/rtclock.c 34 * Purpose: Routines for handling the machine dependent 35 * real-time clock. Historically, this clock is 36 * generated by the Intel 8254 Programmable Interval 37 * Timer, but local apic timers are now used for 38 * this purpose with the master time reference being 39 * the cpu clock counted by the timestamp MSR. 40 */ 41 42#include <platforms.h> 43 44#include <mach/mach_types.h> 45 46#include <kern/cpu_data.h> 47#include <kern/cpu_number.h> 48#include <kern/clock.h> 49#include <kern/host_notify.h> 50#include <kern/macro_help.h> 51#include <kern/misc_protos.h> 52#include <kern/spl.h> 53#include <kern/assert.h> 54#include <kern/timer_queue.h> 55#include <mach/vm_prot.h> 56#include <vm/pmap.h> 57#include <vm/vm_kern.h> /* for kernel_map */ 58#include <architecture/i386/pio.h> 59#include <i386/machine_cpu.h> 60#include <i386/cpuid.h> 61#include <i386/cpu_threads.h> 62#include <i386/mp.h> 63#include <i386/machine_routines.h> 64#include <i386/pal_routines.h> 65#include <i386/proc_reg.h> 66#include <i386/misc_protos.h> 67#include <pexpert/pexpert.h> 68#include <machine/limits.h> 69#include <machine/commpage.h> 70#include <sys/kdebug.h> 71#include <i386/tsc.h> 72#include <i386/rtclock_protos.h> 73#define UI_CPUFREQ_ROUNDING_FACTOR 10000000 74 75int rtclock_config(void); 76 77int rtclock_init(void); 78 79uint64_t tsc_rebase_abs_time = 0; 80 81static void rtc_set_timescale(uint64_t cycles); 82static uint64_t rtc_export_speed(uint64_t cycles); 83 84void 85rtc_timer_start(void) 86{ 87 /* 88 * Force a complete re-evaluation of timer deadlines. 89 */ 90 x86_lcpu()->rtcDeadline = EndOfAllTime; 91 timer_resync_deadlines(); 92} 93 94static inline uint32_t 95_absolutetime_to_microtime(uint64_t abstime, clock_sec_t *secs, clock_usec_t *microsecs) 96{ 97 uint32_t remain; 98 *secs = abstime / (uint64_t)NSEC_PER_SEC; 99 remain = (uint32_t)(abstime % (uint64_t)NSEC_PER_SEC); 100 *microsecs = remain / NSEC_PER_USEC; 101 return remain; 102} 103 104static inline void 105_absolutetime_to_nanotime(uint64_t abstime, clock_sec_t *secs, clock_usec_t *nanosecs) 106{ 107 *secs = abstime / (uint64_t)NSEC_PER_SEC; 108 *nanosecs = (clock_usec_t)(abstime % (uint64_t)NSEC_PER_SEC); 109} 110 111/* 112 * Configure the real-time clock device. Return success (1) 113 * or failure (0). 114 */ 115 116int 117rtclock_config(void) 118{ 119 /* nothing to do */ 120 return (1); 121} 122 123 124/* 125 * Nanotime/mach_absolutime_time 126 * ----------------------------- 127 * The timestamp counter (TSC) - which counts cpu clock cycles and can be read 128 * efficiently by the kernel and in userspace - is the reference for all timing. 129 * The cpu clock rate is platform-dependent and may stop or be reset when the 130 * processor is napped/slept. As a result, nanotime is the software abstraction 131 * used to maintain a monotonic clock, adjusted from an outside reference as needed. 132 * 133 * The kernel maintains nanotime information recording: 134 * - the ratio of tsc to nanoseconds 135 * with this ratio expressed as a 32-bit scale and shift 136 * (power of 2 divider); 137 * - { tsc_base, ns_base } pair of corresponding timestamps. 138 * 139 * The tuple {tsc_base, ns_base, scale, shift} is exported in the commpage 140 * for the userspace nanotime routine to read. 141 * 142 * All of the routines which update the nanotime data are non-reentrant. This must 143 * be guaranteed by the caller. 144 */ 145static inline void 146rtc_nanotime_set_commpage(pal_rtc_nanotime_t *rntp) 147{ 148 commpage_set_nanotime(rntp->tsc_base, rntp->ns_base, rntp->scale, rntp->shift); 149} 150 151/* 152 * rtc_nanotime_init: 153 * 154 * Intialize the nanotime info from the base time. 155 */ 156static inline void 157_rtc_nanotime_init(pal_rtc_nanotime_t *rntp, uint64_t base) 158{ 159 uint64_t tsc = rdtsc64(); 160 161 _pal_rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp); 162} 163 164static void 165rtc_nanotime_init(uint64_t base) 166{ 167 _rtc_nanotime_init(&pal_rtc_nanotime_info, base); 168 rtc_nanotime_set_commpage(&pal_rtc_nanotime_info); 169} 170 171/* 172 * rtc_nanotime_init_commpage: 173 * 174 * Call back from the commpage initialization to 175 * cause the commpage data to be filled in once the 176 * commpages have been created. 177 */ 178void 179rtc_nanotime_init_commpage(void) 180{ 181 spl_t s = splclock(); 182 183 rtc_nanotime_set_commpage(&pal_rtc_nanotime_info); 184 splx(s); 185} 186 187/* 188 * rtc_nanotime_read: 189 * 190 * Returns the current nanotime value, accessable from any 191 * context. 192 */ 193static inline uint64_t 194rtc_nanotime_read(void) 195{ 196 return _rtc_nanotime_read(&pal_rtc_nanotime_info); 197} 198 199/* 200 * rtc_clock_napped: 201 * 202 * Invoked from power management when we exit from a low C-State (>= C4) 203 * and the TSC has stopped counting. The nanotime data is updated according 204 * to the provided value which represents the new value for nanotime. 205 */ 206void 207rtc_clock_napped(uint64_t base, uint64_t tsc_base) 208{ 209 pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; 210 uint64_t oldnsecs; 211 uint64_t newnsecs; 212 uint64_t tsc; 213 214 assert(!ml_get_interrupts_enabled()); 215 tsc = rdtsc64(); 216 oldnsecs = rntp->ns_base + _rtc_tsc_to_nanoseconds(tsc - rntp->tsc_base, rntp); 217 newnsecs = base + _rtc_tsc_to_nanoseconds(tsc - tsc_base, rntp); 218 219 /* 220 * Only update the base values if time using the new base values 221 * is later than the time using the old base values. 222 */ 223 if (oldnsecs < newnsecs) { 224 _pal_rtc_nanotime_store(tsc_base, base, rntp->scale, rntp->shift, rntp); 225 rtc_nanotime_set_commpage(rntp); 226 } 227} 228 229/* 230 * Invoked from power management to correct the SFLM TSC entry drift problem: 231 * a small delta is added to the tsc_base. This is equivalent to nudgin time 232 * backwards. We require this to be on the order of a TSC quantum which won't 233 * cause callers of mach_absolute_time() to see time going backwards! 234 */ 235void 236rtc_clock_adjust(uint64_t tsc_base_delta) 237{ 238 pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; 239 240 assert(!ml_get_interrupts_enabled()); 241 assert(tsc_base_delta < 100ULL); /* i.e. it's small */ 242 _rtc_nanotime_adjust(tsc_base_delta, rntp); 243 rtc_nanotime_set_commpage(rntp); 244} 245 246void 247rtc_clock_stepping(__unused uint32_t new_frequency, 248 __unused uint32_t old_frequency) 249{ 250 panic("rtc_clock_stepping unsupported"); 251} 252 253void 254rtc_clock_stepped(__unused uint32_t new_frequency, 255 __unused uint32_t old_frequency) 256{ 257 panic("rtc_clock_stepped unsupported"); 258} 259 260/* 261 * rtc_sleep_wakeup: 262 * 263 * Invoked from power management when we have awoken from a sleep (S3) 264 * and the TSC has been reset, or from Deep Idle (S0) sleep when the TSC 265 * has progressed. The nanotime data is updated based on the passed-in value. 266 * 267 * The caller must guarantee non-reentrancy. 268 */ 269void 270rtc_sleep_wakeup( 271 uint64_t base) 272{ 273 /* Set fixed configuration for lapic timers */ 274 rtc_timer->config(); 275 276 /* 277 * Reset nanotime. 278 * The timestamp counter will have been reset 279 * but nanotime (uptime) marches onward. 280 */ 281 rtc_nanotime_init(base); 282} 283 284/* 285 * Initialize the real-time clock device. 286 * In addition, various variables used to support the clock are initialized. 287 */ 288int 289rtclock_init(void) 290{ 291 uint64_t cycles; 292 293 assert(!ml_get_interrupts_enabled()); 294 295 if (cpu_number() == master_cpu) { 296 297 assert(tscFreq); 298 rtc_set_timescale(tscFreq); 299 300 /* 301 * Adjust and set the exported cpu speed. 302 */ 303 cycles = rtc_export_speed(tscFreq); 304 305 /* 306 * Set min/max to actual. 307 * ACPI may update these later if speed-stepping is detected. 308 */ 309 gPEClockFrequencyInfo.cpu_frequency_min_hz = cycles; 310 gPEClockFrequencyInfo.cpu_frequency_max_hz = cycles; 311 312 rtc_timer_init(); 313 clock_timebase_init(); 314 ml_init_lock_timeout(); 315 ml_init_delay_spin_threshold(10); 316 } 317 318 /* Set fixed configuration for lapic timers */ 319 rtc_timer->config(); 320 rtc_timer_start(); 321 322 return (1); 323} 324 325// utility routine 326// Code to calculate how many processor cycles are in a second... 327 328static void 329rtc_set_timescale(uint64_t cycles) 330{ 331 pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info; 332 uint32_t shift = 0; 333 334 /* the "scale" factor will overflow unless cycles>SLOW_TSC_THRESHOLD */ 335 336 while ( cycles <= SLOW_TSC_THRESHOLD) { 337 shift++; 338 cycles <<= 1; 339 } 340 341 if ( shift != 0 ) 342 printf("Slow TSC, rtc_nanotime.shift == %d\n", shift); 343 344 rntp->scale = (uint32_t)(((uint64_t)NSEC_PER_SEC << 32) / cycles); 345 346 rntp->shift = shift; 347 348 /* 349 * On some platforms, the TSC is not reset at warm boot. But the 350 * rebase time must be relative to the current boot so we can't use 351 * mach_absolute_time(). Instead, we convert the TSC delta since boot 352 * to nanoseconds. 353 */ 354 if (tsc_rebase_abs_time == 0) 355 tsc_rebase_abs_time = _rtc_tsc_to_nanoseconds( 356 rdtsc64() - tsc_at_boot, rntp); 357 358 rtc_nanotime_init(0); 359} 360 361static uint64_t 362rtc_export_speed(uint64_t cyc_per_sec) 363{ 364 uint64_t cycles; 365 366 /* Round: */ 367 cycles = ((cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR/2)) 368 / UI_CPUFREQ_ROUNDING_FACTOR) 369 * UI_CPUFREQ_ROUNDING_FACTOR; 370 371 /* 372 * Set current measured speed. 373 */ 374 if (cycles >= 0x100000000ULL) { 375 gPEClockFrequencyInfo.cpu_clock_rate_hz = 0xFFFFFFFFUL; 376 } else { 377 gPEClockFrequencyInfo.cpu_clock_rate_hz = (unsigned long)cycles; 378 } 379 gPEClockFrequencyInfo.cpu_frequency_hz = cycles; 380 381 kprintf("[RTCLOCK] frequency %llu (%llu)\n", cycles, cyc_per_sec); 382 return(cycles); 383} 384 385void 386clock_get_system_microtime( 387 clock_sec_t *secs, 388 clock_usec_t *microsecs) 389{ 390 uint64_t now = rtc_nanotime_read(); 391 392 _absolutetime_to_microtime(now, secs, microsecs); 393} 394 395void 396clock_get_system_nanotime( 397 clock_sec_t *secs, 398 clock_nsec_t *nanosecs) 399{ 400 uint64_t now = rtc_nanotime_read(); 401 402 _absolutetime_to_nanotime(now, secs, nanosecs); 403} 404 405void 406clock_gettimeofday_set_commpage( 407 uint64_t abstime, 408 uint64_t epoch, 409 uint64_t offset, 410 clock_sec_t *secs, 411 clock_usec_t *microsecs) 412{ 413 uint64_t now = abstime + offset; 414 uint32_t remain; 415 416 remain = _absolutetime_to_microtime(now, secs, microsecs); 417 418 *secs += (clock_sec_t)epoch; 419 420 commpage_set_timestamp(abstime - remain, *secs); 421} 422 423void 424clock_timebase_info( 425 mach_timebase_info_t info) 426{ 427 info->numer = info->denom = 1; 428} 429 430/* 431 * Real-time clock device interrupt. 432 */ 433void 434rtclock_intr( 435 x86_saved_state_t *tregs) 436{ 437 uint64_t rip; 438 boolean_t user_mode = FALSE; 439 440 assert(get_preemption_level() > 0); 441 assert(!ml_get_interrupts_enabled()); 442 443 if (is_saved_state64(tregs) == TRUE) { 444 x86_saved_state64_t *regs; 445 446 regs = saved_state64(tregs); 447 448 if (regs->isf.cs & 0x03) 449 user_mode = TRUE; 450 rip = regs->isf.rip; 451 } else { 452 x86_saved_state32_t *regs; 453 454 regs = saved_state32(tregs); 455 456 if (regs->cs & 0x03) 457 user_mode = TRUE; 458 rip = regs->eip; 459 } 460 461 /* call the generic etimer */ 462 timer_intr(user_mode, rip); 463} 464 465 466/* 467 * Request timer pop from the hardware 468 */ 469 470uint64_t 471setPop( 472 uint64_t time) 473{ 474 uint64_t now; 475 uint64_t pop; 476 477 /* 0 and EndOfAllTime are special-cases for "clear the timer" */ 478 if (time == 0 || time == EndOfAllTime ) { 479 time = EndOfAllTime; 480 now = 0; 481 pop = rtc_timer->set(0, 0); 482 } else { 483 now = rtc_nanotime_read(); /* The time in nanoseconds */ 484 pop = rtc_timer->set(time, now); 485 } 486 487 /* Record requested and actual deadlines set */ 488 x86_lcpu()->rtcDeadline = time; 489 x86_lcpu()->rtcPop = pop; 490 491 return pop - now; 492} 493 494uint64_t 495mach_absolute_time(void) 496{ 497 return rtc_nanotime_read(); 498} 499 500void 501clock_interval_to_absolutetime_interval( 502 uint32_t interval, 503 uint32_t scale_factor, 504 uint64_t *result) 505{ 506 *result = (uint64_t)interval * scale_factor; 507} 508 509void 510absolutetime_to_microtime( 511 uint64_t abstime, 512 clock_sec_t *secs, 513 clock_usec_t *microsecs) 514{ 515 _absolutetime_to_microtime(abstime, secs, microsecs); 516} 517 518void 519absolutetime_to_nanotime( 520 uint64_t abstime, 521 clock_sec_t *secs, 522 clock_nsec_t *nanosecs) 523{ 524 _absolutetime_to_nanotime(abstime, secs, nanosecs); 525} 526 527void 528nanotime_to_absolutetime( 529 clock_sec_t secs, 530 clock_nsec_t nanosecs, 531 uint64_t *result) 532{ 533 *result = ((uint64_t)secs * NSEC_PER_SEC) + nanosecs; 534} 535 536void 537absolutetime_to_nanoseconds( 538 uint64_t abstime, 539 uint64_t *result) 540{ 541 *result = abstime; 542} 543 544void 545nanoseconds_to_absolutetime( 546 uint64_t nanoseconds, 547 uint64_t *result) 548{ 549 *result = nanoseconds; 550} 551 552void 553machine_delay_until( 554 uint64_t interval, 555 uint64_t deadline) 556{ 557 (void)interval; 558 while (mach_absolute_time() < deadline) { 559 cpu_pause(); 560 } 561} 562