kern_tc.c revision 227747
1/*- 2 * ---------------------------------------------------------------------------- 3 * "THE BEER-WARE LICENSE" (Revision 42): 4 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you 5 * can do whatever you want with this stuff. If we meet some day, and you think 6 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 7 * ---------------------------------------------------------------------------- 8 * 9 * Copyright (c) 2011 The FreeBSD Foundation 10 * All rights reserved. 11 * 12 * Portions of this software were developed by Julien Ridoux at the University 13 * of Melbourne under sponsorship from the FreeBSD Foundation. 14 */ 15 16#include <sys/cdefs.h> 17__FBSDID("$FreeBSD: head/sys/kern/kern_tc.c 227747 2011-11-20 05:32:12Z lstewart $"); 18 19#include "opt_ntp.h" 20#include "opt_ffclock.h" 21 22#include <sys/param.h> 23#include <sys/kernel.h> 24#ifdef FFCLOCK 25#include <sys/lock.h> 26#include <sys/mutex.h> 27#endif 28#include <sys/sysctl.h> 29#include <sys/syslog.h> 30#include <sys/systm.h> 31#ifdef FFCLOCK 32#include <sys/timeffc.h> 33#endif 34#include <sys/timepps.h> 35#include <sys/timetc.h> 36#include <sys/timex.h> 37 38/* 39 * A large step happens on boot. This constant detects such steps. 40 * It is relatively small so that ntp_update_second gets called enough 41 * in the typical 'missed a couple of seconds' case, but doesn't loop 42 * forever when the time step is large. 43 */ 44#define LARGE_STEP 200 45 46/* 47 * Implement a dummy timecounter which we can use until we get a real one 48 * in the air. This allows the console and other early stuff to use 49 * time services. 50 */ 51 52static u_int 53dummy_get_timecount(struct timecounter *tc) 54{ 55 static u_int now; 56 57 return (++now); 58} 59 60static struct timecounter dummy_timecounter = { 61 dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000 62}; 63 64struct timehands { 65 /* These fields must be initialized by the driver. */ 66 struct timecounter *th_counter; 67 int64_t th_adjustment; 68 uint64_t th_scale; 69 u_int th_offset_count; 70 struct bintime th_offset; 71 struct timeval th_microtime; 72 struct timespec th_nanotime; 73 /* Fields not to be copied in tc_windup start with th_generation. */ 74 volatile u_int th_generation; 75 struct timehands *th_next; 76}; 77 78static struct timehands th0; 79static struct timehands th9 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th0}; 80static struct timehands th8 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th9}; 81static struct timehands th7 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th8}; 82static struct timehands th6 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th7}; 83static struct timehands th5 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th6}; 84static struct timehands th4 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th5}; 85static struct timehands th3 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th4}; 86static struct timehands th2 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th3}; 87static struct timehands th1 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th2}; 88static struct timehands th0 = { 89 &dummy_timecounter, 90 0, 91 (uint64_t)-1 / 1000000, 92 0, 93 {1, 0}, 94 {0, 0}, 95 {0, 0}, 96 1, 97 &th1 98}; 99 100static struct timehands *volatile timehands = &th0; 101struct timecounter *timecounter = &dummy_timecounter; 102static struct timecounter *timecounters = &dummy_timecounter; 103 104int tc_min_ticktock_freq = 1; 105 106time_t time_second = 1; 107time_t time_uptime = 1; 108 109struct bintime boottimebin; 110struct timeval boottime; 111static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS); 112SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD, 113 NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime"); 114 115SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, ""); 116static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, ""); 117 118static int timestepwarnings; 119SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW, 120 ×tepwarnings, 0, "Log time steps"); 121 122static void tc_windup(void); 123static void cpu_tick_calibrate(int); 124 125static int 126sysctl_kern_boottime(SYSCTL_HANDLER_ARGS) 127{ 128#ifdef SCTL_MASK32 129 int tv[2]; 130 131 if (req->flags & SCTL_MASK32) { 132 tv[0] = boottime.tv_sec; 133 tv[1] = boottime.tv_usec; 134 return SYSCTL_OUT(req, tv, sizeof(tv)); 135 } else 136#endif 137 return SYSCTL_OUT(req, &boottime, sizeof(boottime)); 138} 139 140static int 141sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS) 142{ 143 u_int ncount; 144 struct timecounter *tc = arg1; 145 146 ncount = tc->tc_get_timecount(tc); 147 return sysctl_handle_int(oidp, &ncount, 0, req); 148} 149 150static int 151sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS) 152{ 153 uint64_t freq; 154 struct timecounter *tc = arg1; 155 156 freq = tc->tc_frequency; 157 return sysctl_handle_64(oidp, &freq, 0, req); 158} 159 160/* 161 * Return the difference between the timehands' counter value now and what 162 * was when we copied it to the timehands' offset_count. 163 */ 164static __inline u_int 165tc_delta(struct timehands *th) 166{ 167 struct timecounter *tc; 168 169 tc = th->th_counter; 170 return ((tc->tc_get_timecount(tc) - th->th_offset_count) & 171 tc->tc_counter_mask); 172} 173 174/* 175 * Functions for reading the time. We have to loop until we are sure that 176 * the timehands that we operated on was not updated under our feet. See 177 * the comment in <sys/time.h> for a description of these 12 functions. 178 */ 179 180#ifdef FFCLOCK 181static void 182fbclock_binuptime(struct bintime *bt) 183{ 184 struct timehands *th; 185 unsigned int gen; 186 187 do { 188 th = timehands; 189 gen = th->th_generation; 190 *bt = th->th_offset; 191 bintime_addx(bt, th->th_scale * tc_delta(th)); 192 } while (gen == 0 || gen != th->th_generation); 193} 194 195static void 196fbclock_nanouptime(struct timespec *tsp) 197{ 198 struct bintime bt; 199 200 binuptime(&bt); 201 bintime2timespec(&bt, tsp); 202} 203 204static void 205fbclock_microuptime(struct timeval *tvp) 206{ 207 struct bintime bt; 208 209 binuptime(&bt); 210 bintime2timeval(&bt, tvp); 211} 212 213static void 214fbclock_bintime(struct bintime *bt) 215{ 216 217 binuptime(bt); 218 bintime_add(bt, &boottimebin); 219} 220 221static void 222fbclock_nanotime(struct timespec *tsp) 223{ 224 struct bintime bt; 225 226 bintime(&bt); 227 bintime2timespec(&bt, tsp); 228} 229 230static void 231fbclock_microtime(struct timeval *tvp) 232{ 233 struct bintime bt; 234 235 bintime(&bt); 236 bintime2timeval(&bt, tvp); 237} 238 239static void 240fbclock_getbinuptime(struct bintime *bt) 241{ 242 struct timehands *th; 243 unsigned int gen; 244 245 do { 246 th = timehands; 247 gen = th->th_generation; 248 *bt = th->th_offset; 249 } while (gen == 0 || gen != th->th_generation); 250} 251 252static void 253fbclock_getnanouptime(struct timespec *tsp) 254{ 255 struct timehands *th; 256 unsigned int gen; 257 258 do { 259 th = timehands; 260 gen = th->th_generation; 261 bintime2timespec(&th->th_offset, tsp); 262 } while (gen == 0 || gen != th->th_generation); 263} 264 265static void 266fbclock_getmicrouptime(struct timeval *tvp) 267{ 268 struct timehands *th; 269 unsigned int gen; 270 271 do { 272 th = timehands; 273 gen = th->th_generation; 274 bintime2timeval(&th->th_offset, tvp); 275 } while (gen == 0 || gen != th->th_generation); 276} 277 278static void 279fbclock_getbintime(struct bintime *bt) 280{ 281 struct timehands *th; 282 unsigned int gen; 283 284 do { 285 th = timehands; 286 gen = th->th_generation; 287 *bt = th->th_offset; 288 } while (gen == 0 || gen != th->th_generation); 289 bintime_add(bt, &boottimebin); 290} 291 292static void 293fbclock_getnanotime(struct timespec *tsp) 294{ 295 struct timehands *th; 296 unsigned int gen; 297 298 do { 299 th = timehands; 300 gen = th->th_generation; 301 *tsp = th->th_nanotime; 302 } while (gen == 0 || gen != th->th_generation); 303} 304 305static void 306fbclock_getmicrotime(struct timeval *tvp) 307{ 308 struct timehands *th; 309 unsigned int gen; 310 311 do { 312 th = timehands; 313 gen = th->th_generation; 314 *tvp = th->th_microtime; 315 } while (gen == 0 || gen != th->th_generation); 316} 317#else /* !FFCLOCK */ 318void 319binuptime(struct bintime *bt) 320{ 321 struct timehands *th; 322 u_int gen; 323 324 do { 325 th = timehands; 326 gen = th->th_generation; 327 *bt = th->th_offset; 328 bintime_addx(bt, th->th_scale * tc_delta(th)); 329 } while (gen == 0 || gen != th->th_generation); 330} 331 332void 333nanouptime(struct timespec *tsp) 334{ 335 struct bintime bt; 336 337 binuptime(&bt); 338 bintime2timespec(&bt, tsp); 339} 340 341void 342microuptime(struct timeval *tvp) 343{ 344 struct bintime bt; 345 346 binuptime(&bt); 347 bintime2timeval(&bt, tvp); 348} 349 350void 351bintime(struct bintime *bt) 352{ 353 354 binuptime(bt); 355 bintime_add(bt, &boottimebin); 356} 357 358void 359nanotime(struct timespec *tsp) 360{ 361 struct bintime bt; 362 363 bintime(&bt); 364 bintime2timespec(&bt, tsp); 365} 366 367void 368microtime(struct timeval *tvp) 369{ 370 struct bintime bt; 371 372 bintime(&bt); 373 bintime2timeval(&bt, tvp); 374} 375 376void 377getbinuptime(struct bintime *bt) 378{ 379 struct timehands *th; 380 u_int gen; 381 382 do { 383 th = timehands; 384 gen = th->th_generation; 385 *bt = th->th_offset; 386 } while (gen == 0 || gen != th->th_generation); 387} 388 389void 390getnanouptime(struct timespec *tsp) 391{ 392 struct timehands *th; 393 u_int gen; 394 395 do { 396 th = timehands; 397 gen = th->th_generation; 398 bintime2timespec(&th->th_offset, tsp); 399 } while (gen == 0 || gen != th->th_generation); 400} 401 402void 403getmicrouptime(struct timeval *tvp) 404{ 405 struct timehands *th; 406 u_int gen; 407 408 do { 409 th = timehands; 410 gen = th->th_generation; 411 bintime2timeval(&th->th_offset, tvp); 412 } while (gen == 0 || gen != th->th_generation); 413} 414 415void 416getbintime(struct bintime *bt) 417{ 418 struct timehands *th; 419 u_int gen; 420 421 do { 422 th = timehands; 423 gen = th->th_generation; 424 *bt = th->th_offset; 425 } while (gen == 0 || gen != th->th_generation); 426 bintime_add(bt, &boottimebin); 427} 428 429void 430getnanotime(struct timespec *tsp) 431{ 432 struct timehands *th; 433 u_int gen; 434 435 do { 436 th = timehands; 437 gen = th->th_generation; 438 *tsp = th->th_nanotime; 439 } while (gen == 0 || gen != th->th_generation); 440} 441 442void 443getmicrotime(struct timeval *tvp) 444{ 445 struct timehands *th; 446 u_int gen; 447 448 do { 449 th = timehands; 450 gen = th->th_generation; 451 *tvp = th->th_microtime; 452 } while (gen == 0 || gen != th->th_generation); 453} 454#endif /* FFCLOCK */ 455 456#ifdef FFCLOCK 457/* 458 * Support for feed-forward synchronization algorithms. This is heavily inspired 459 * by the timehands mechanism but kept independent from it. *_windup() functions 460 * have some connection to avoid accessing the timecounter hardware more than 461 * necessary. 462 */ 463 464int sysclock_active = SYSCLOCK_FBCK; 465 466/* Feed-forward clock estimates kept updated by the synchronization daemon. */ 467struct ffclock_estimate ffclock_estimate; 468struct bintime ffclock_boottime; /* Feed-forward boot time estimate. */ 469uint32_t ffclock_status; /* Feed-forward clock status. */ 470int8_t ffclock_updated; /* New estimates are available. */ 471struct mtx ffclock_mtx; /* Mutex on ffclock_estimate. */ 472 473struct sysclock_ops { 474 int active; 475 void (*binuptime) (struct bintime *bt); 476 void (*nanouptime) (struct timespec *tsp); 477 void (*microuptime) (struct timeval *tvp); 478 void (*bintime) (struct bintime *bt); 479 void (*nanotime) (struct timespec *tsp); 480 void (*microtime) (struct timeval *tvp); 481 void (*getbinuptime) (struct bintime *bt); 482 void (*getnanouptime) (struct timespec *tsp); 483 void (*getmicrouptime) (struct timeval *tvp); 484 void (*getbintime) (struct bintime *bt); 485 void (*getnanotime) (struct timespec *tsp); 486 void (*getmicrotime) (struct timeval *tvp); 487}; 488 489static struct sysclock_ops sysclock = { 490 .active = SYSCLOCK_FBCK, 491 .binuptime = fbclock_binuptime, 492 .nanouptime = fbclock_nanouptime, 493 .microuptime = fbclock_microuptime, 494 .bintime = fbclock_bintime, 495 .nanotime = fbclock_nanotime, 496 .microtime = fbclock_microtime, 497 .getbinuptime = fbclock_getbinuptime, 498 .getnanouptime = fbclock_getnanouptime, 499 .getmicrouptime = fbclock_getmicrouptime, 500 .getbintime = fbclock_getbintime, 501 .getnanotime = fbclock_getnanotime, 502 .getmicrotime = fbclock_getmicrotime 503}; 504 505struct fftimehands { 506 struct ffclock_estimate cest; 507 struct bintime tick_time; 508 struct bintime tick_time_lerp; 509 ffcounter tick_ffcount; 510 uint64_t period_lerp; 511 volatile uint8_t gen; 512 struct fftimehands *next; 513}; 514 515#define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x)) 516 517static struct fftimehands ffth[10]; 518static struct fftimehands *volatile fftimehands = ffth; 519 520static void 521ffclock_init(void) 522{ 523 struct fftimehands *cur; 524 struct fftimehands *last; 525 526 memset(ffth, 0, sizeof(ffth)); 527 528 last = ffth + NUM_ELEMENTS(ffth) - 1; 529 for (cur = ffth; cur < last; cur++) 530 cur->next = cur + 1; 531 last->next = ffth; 532 533 ffclock_updated = 0; 534 ffclock_status = FFCLOCK_STA_UNSYNC; 535 mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF); 536} 537 538/* 539 * Reset the feed-forward clock estimates. Called from inittodr() to get things 540 * kick started and uses the timecounter nominal frequency as a first period 541 * estimate. Note: this function may be called several time just after boot. 542 * Note: this is the only function that sets the value of boot time for the 543 * monotonic (i.e. uptime) version of the feed-forward clock. 544 */ 545void 546ffclock_reset_clock(struct timespec *ts) 547{ 548 struct timecounter *tc; 549 struct ffclock_estimate cest; 550 551 tc = timehands->th_counter; 552 memset(&cest, 0, sizeof(struct ffclock_estimate)); 553 554 timespec2bintime(ts, &ffclock_boottime); 555 timespec2bintime(ts, &(cest.update_time)); 556 ffclock_read_counter(&cest.update_ffcount); 557 cest.leapsec_next = 0; 558 cest.period = ((1ULL << 63) / tc->tc_frequency) << 1; 559 cest.errb_abs = 0; 560 cest.errb_rate = 0; 561 cest.status = FFCLOCK_STA_UNSYNC; 562 cest.leapsec_total = 0; 563 cest.leapsec = 0; 564 565 mtx_lock(&ffclock_mtx); 566 bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate)); 567 ffclock_updated = INT8_MAX; 568 mtx_unlock(&ffclock_mtx); 569 570 printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name, 571 (unsigned long long)tc->tc_frequency, (long)ts->tv_sec, 572 (unsigned long)ts->tv_nsec); 573} 574 575/* 576 * Sub-routine to convert a time interval measured in RAW counter units to time 577 * in seconds stored in bintime format. 578 * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be 579 * larger than the max value of u_int (on 32 bit architecture). Loop to consume 580 * extra cycles. 581 */ 582static void 583ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt) 584{ 585 struct bintime bt2; 586 ffcounter delta, delta_max; 587 588 delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1; 589 bintime_clear(bt); 590 do { 591 if (ffdelta > delta_max) 592 delta = delta_max; 593 else 594 delta = ffdelta; 595 bt2.sec = 0; 596 bt2.frac = period; 597 bintime_mul(&bt2, (unsigned int)delta); 598 bintime_add(bt, &bt2); 599 ffdelta -= delta; 600 } while (ffdelta > 0); 601} 602 603/* 604 * Update the fftimehands. 605 * Push the tick ffcount and time(s) forward based on current clock estimate. 606 * The conversion from ffcounter to bintime relies on the difference clock 607 * principle, whose accuracy relies on computing small time intervals. If a new 608 * clock estimate has been passed by the synchronisation daemon, make it 609 * current, and compute the linear interpolation for monotonic time if needed. 610 */ 611static void 612ffclock_windup(unsigned int delta) 613{ 614 struct ffclock_estimate *cest; 615 struct fftimehands *ffth; 616 struct bintime bt, gap_lerp; 617 ffcounter ffdelta; 618 uint64_t frac; 619 unsigned int polling; 620 uint8_t forward_jump, ogen; 621 622 /* 623 * Pick the next timehand, copy current ffclock estimates and move tick 624 * times and counter forward. 625 */ 626 forward_jump = 0; 627 ffth = fftimehands->next; 628 ogen = ffth->gen; 629 ffth->gen = 0; 630 cest = &ffth->cest; 631 bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate)); 632 ffdelta = (ffcounter)delta; 633 ffth->period_lerp = fftimehands->period_lerp; 634 635 ffth->tick_time = fftimehands->tick_time; 636 ffclock_convert_delta(ffdelta, cest->period, &bt); 637 bintime_add(&ffth->tick_time, &bt); 638 639 ffth->tick_time_lerp = fftimehands->tick_time_lerp; 640 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt); 641 bintime_add(&ffth->tick_time_lerp, &bt); 642 643 ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta; 644 645 /* 646 * Assess the status of the clock, if the last update is too old, it is 647 * likely the synchronisation daemon is dead and the clock is free 648 * running. 649 */ 650 if (ffclock_updated == 0) { 651 ffdelta = ffth->tick_ffcount - cest->update_ffcount; 652 ffclock_convert_delta(ffdelta, cest->period, &bt); 653 if (bt.sec > 2 * FFCLOCK_SKM_SCALE) 654 ffclock_status |= FFCLOCK_STA_UNSYNC; 655 } 656 657 /* 658 * If available, grab updated clock estimates and make them current. 659 * Recompute time at this tick using the updated estimates. The clock 660 * estimates passed the feed-forward synchronisation daemon may result 661 * in time conversion that is not monotonically increasing (just after 662 * the update). time_lerp is a particular linear interpolation over the 663 * synchronisation algo polling period that ensures monotonicity for the 664 * clock ids requesting it. 665 */ 666 if (ffclock_updated > 0) { 667 bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate)); 668 ffdelta = ffth->tick_ffcount - cest->update_ffcount; 669 ffth->tick_time = cest->update_time; 670 ffclock_convert_delta(ffdelta, cest->period, &bt); 671 bintime_add(&ffth->tick_time, &bt); 672 673 /* ffclock_reset sets ffclock_updated to INT8_MAX */ 674 if (ffclock_updated == INT8_MAX) 675 ffth->tick_time_lerp = ffth->tick_time; 676 677 if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >)) 678 forward_jump = 1; 679 else 680 forward_jump = 0; 681 682 bintime_clear(&gap_lerp); 683 if (forward_jump) { 684 gap_lerp = ffth->tick_time; 685 bintime_sub(&gap_lerp, &ffth->tick_time_lerp); 686 } else { 687 gap_lerp = ffth->tick_time_lerp; 688 bintime_sub(&gap_lerp, &ffth->tick_time); 689 } 690 691 /* 692 * The reset from the RTC clock may be far from accurate, and 693 * reducing the gap between real time and interpolated time 694 * could take a very long time if the interpolated clock insists 695 * on strict monotonicity. The clock is reset under very strict 696 * conditions (kernel time is known to be wrong and 697 * synchronization daemon has been restarted recently. 698 * ffclock_boottime absorbs the jump to ensure boot time is 699 * correct and uptime functions stay consistent. 700 */ 701 if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) && 702 ((cest->status & FFCLOCK_STA_UNSYNC) == 0) && 703 ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) { 704 if (forward_jump) 705 bintime_add(&ffclock_boottime, &gap_lerp); 706 else 707 bintime_sub(&ffclock_boottime, &gap_lerp); 708 ffth->tick_time_lerp = ffth->tick_time; 709 bintime_clear(&gap_lerp); 710 } 711 712 ffclock_status = cest->status; 713 ffth->period_lerp = cest->period; 714 715 /* 716 * Compute corrected period used for the linear interpolation of 717 * time. The rate of linear interpolation is capped to 5000PPM 718 * (5ms/s). 719 */ 720 if (bintime_isset(&gap_lerp)) { 721 ffdelta = cest->update_ffcount; 722 ffdelta -= fftimehands->cest.update_ffcount; 723 ffclock_convert_delta(ffdelta, cest->period, &bt); 724 polling = bt.sec; 725 bt.sec = 0; 726 bt.frac = 5000000 * (uint64_t)18446744073LL; 727 bintime_mul(&bt, polling); 728 if (bintime_cmp(&gap_lerp, &bt, >)) 729 gap_lerp = bt; 730 731 /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */ 732 frac = 0; 733 if (gap_lerp.sec > 0) { 734 frac -= 1; 735 frac /= ffdelta / gap_lerp.sec; 736 } 737 frac += gap_lerp.frac / ffdelta; 738 739 if (forward_jump) 740 ffth->period_lerp += frac; 741 else 742 ffth->period_lerp -= frac; 743 } 744 745 ffclock_updated = 0; 746 } 747 if (++ogen == 0) 748 ogen = 1; 749 ffth->gen = ogen; 750 fftimehands = ffth; 751} 752 753/* 754 * Adjust the fftimehands when the timecounter is changed. Stating the obvious, 755 * the old and new hardware counter cannot be read simultaneously. tc_windup() 756 * does read the two counters 'back to back', but a few cycles are effectively 757 * lost, and not accumulated in tick_ffcount. This is a fairly radical 758 * operation for a feed-forward synchronization daemon, and it is its job to not 759 * pushing irrelevant data to the kernel. Because there is no locking here, 760 * simply force to ignore pending or next update to give daemon a chance to 761 * realize the counter has changed. 762 */ 763static void 764ffclock_change_tc(struct timehands *th) 765{ 766 struct fftimehands *ffth; 767 struct ffclock_estimate *cest; 768 struct timecounter *tc; 769 uint8_t ogen; 770 771 tc = th->th_counter; 772 ffth = fftimehands->next; 773 ogen = ffth->gen; 774 ffth->gen = 0; 775 776 cest = &ffth->cest; 777 bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate)); 778 cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1; 779 cest->errb_abs = 0; 780 cest->errb_rate = 0; 781 cest->status |= FFCLOCK_STA_UNSYNC; 782 783 ffth->tick_ffcount = fftimehands->tick_ffcount; 784 ffth->tick_time_lerp = fftimehands->tick_time_lerp; 785 ffth->tick_time = fftimehands->tick_time; 786 ffth->period_lerp = cest->period; 787 788 /* Do not lock but ignore next update from synchronization daemon. */ 789 ffclock_updated--; 790 791 if (++ogen == 0) 792 ogen = 1; 793 ffth->gen = ogen; 794 fftimehands = ffth; 795} 796 797static void 798change_sysclock(int new_sysclock) 799{ 800 801 sysclock.active = new_sysclock; 802 803 switch (sysclock.active) { 804 case SYSCLOCK_FBCK: 805 sysclock.binuptime = fbclock_binuptime; 806 sysclock.nanouptime = fbclock_nanouptime; 807 sysclock.microuptime = fbclock_microuptime; 808 sysclock.bintime = fbclock_bintime; 809 sysclock.nanotime = fbclock_nanotime; 810 sysclock.microtime = fbclock_microtime; 811 sysclock.getbinuptime = fbclock_getbinuptime; 812 sysclock.getnanouptime = fbclock_getnanouptime; 813 sysclock.getmicrouptime = fbclock_getmicrouptime; 814 sysclock.getbintime = fbclock_getbintime; 815 sysclock.getnanotime = fbclock_getnanotime; 816 sysclock.getmicrotime = fbclock_getmicrotime; 817 break; 818 case SYSCLOCK_FFWD: 819 sysclock.binuptime = ffclock_binuptime; 820 sysclock.nanouptime = ffclock_nanouptime; 821 sysclock.microuptime = ffclock_microuptime; 822 sysclock.bintime = ffclock_bintime; 823 sysclock.nanotime = ffclock_nanotime; 824 sysclock.microtime = ffclock_microtime; 825 sysclock.getbinuptime = ffclock_getbinuptime; 826 sysclock.getnanouptime = ffclock_getnanouptime; 827 sysclock.getmicrouptime = ffclock_getmicrouptime; 828 sysclock.getbintime = ffclock_getbintime; 829 sysclock.getnanotime = ffclock_getnanotime; 830 sysclock.getmicrotime = ffclock_getmicrotime; 831 break; 832 default: 833 break; 834 } 835} 836 837/* 838 * Retrieve feed-forward counter and time of last kernel tick. 839 */ 840void 841ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags) 842{ 843 struct fftimehands *ffth; 844 uint8_t gen; 845 846 /* 847 * No locking but check generation has not changed. Also need to make 848 * sure ffdelta is positive, i.e. ffcount > tick_ffcount. 849 */ 850 do { 851 ffth = fftimehands; 852 gen = ffth->gen; 853 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) 854 *bt = ffth->tick_time_lerp; 855 else 856 *bt = ffth->tick_time; 857 *ffcount = ffth->tick_ffcount; 858 } while (gen == 0 || gen != ffth->gen); 859} 860 861/* 862 * Absolute clock conversion. Low level function to convert ffcounter to 863 * bintime. The ffcounter is converted using the current ffclock period estimate 864 * or the "interpolated period" to ensure monotonicity. 865 * NOTE: this conversion may have been deferred, and the clock updated since the 866 * hardware counter has been read. 867 */ 868void 869ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags) 870{ 871 struct fftimehands *ffth; 872 struct bintime bt2; 873 ffcounter ffdelta; 874 uint8_t gen; 875 876 /* 877 * No locking but check generation has not changed. Also need to make 878 * sure ffdelta is positive, i.e. ffcount > tick_ffcount. 879 */ 880 do { 881 ffth = fftimehands; 882 gen = ffth->gen; 883 if (ffcount > ffth->tick_ffcount) 884 ffdelta = ffcount - ffth->tick_ffcount; 885 else 886 ffdelta = ffth->tick_ffcount - ffcount; 887 888 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) { 889 *bt = ffth->tick_time_lerp; 890 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2); 891 } else { 892 *bt = ffth->tick_time; 893 ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2); 894 } 895 896 if (ffcount > ffth->tick_ffcount) 897 bintime_add(bt, &bt2); 898 else 899 bintime_sub(bt, &bt2); 900 } while (gen == 0 || gen != ffth->gen); 901} 902 903/* 904 * Difference clock conversion. 905 * Low level function to Convert a time interval measured in RAW counter units 906 * into bintime. The difference clock allows measuring small intervals much more 907 * reliably than the absolute clock. 908 */ 909void 910ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt) 911{ 912 struct fftimehands *ffth; 913 uint8_t gen; 914 915 /* No locking but check generation has not changed. */ 916 do { 917 ffth = fftimehands; 918 gen = ffth->gen; 919 ffclock_convert_delta(ffdelta, ffth->cest.period, bt); 920 } while (gen == 0 || gen != ffth->gen); 921} 922 923/* 924 * Access to current ffcounter value. 925 */ 926void 927ffclock_read_counter(ffcounter *ffcount) 928{ 929 struct timehands *th; 930 struct fftimehands *ffth; 931 unsigned int gen, delta; 932 933 /* 934 * ffclock_windup() called from tc_windup(), safe to rely on 935 * th->th_generation only, for correct delta and ffcounter. 936 */ 937 do { 938 th = timehands; 939 gen = th->th_generation; 940 ffth = fftimehands; 941 delta = tc_delta(th); 942 *ffcount = ffth->tick_ffcount; 943 } while (gen == 0 || gen != th->th_generation); 944 945 *ffcount += delta; 946} 947 948void 949binuptime(struct bintime *bt) 950{ 951 952 sysclock.binuptime(bt); 953} 954 955void 956nanouptime(struct timespec *tsp) 957{ 958 959 sysclock.nanouptime(tsp); 960} 961 962void 963microuptime(struct timeval *tvp) 964{ 965 966 sysclock.microuptime(tvp); 967} 968 969void 970bintime(struct bintime *bt) 971{ 972 973 sysclock.bintime(bt); 974} 975 976void 977nanotime(struct timespec *tsp) 978{ 979 980 sysclock.nanotime(tsp); 981} 982 983void 984microtime(struct timeval *tvp) 985{ 986 987 sysclock.microtime(tvp); 988} 989 990void 991getbinuptime(struct bintime *bt) 992{ 993 994 sysclock.getbinuptime(bt); 995} 996 997void 998getnanouptime(struct timespec *tsp) 999{ 1000 1001 sysclock.getnanouptime(tsp); 1002} 1003 1004void 1005getmicrouptime(struct timeval *tvp) 1006{ 1007 1008 sysclock.getmicrouptime(tvp); 1009} 1010 1011void 1012getbintime(struct bintime *bt) 1013{ 1014 1015 sysclock.getbintime(bt); 1016} 1017 1018void 1019getnanotime(struct timespec *tsp) 1020{ 1021 1022 sysclock.getnanotime(tsp); 1023} 1024 1025void 1026getmicrotime(struct timeval *tvp) 1027{ 1028 1029 sysclock.getmicrouptime(tvp); 1030} 1031#endif /* FFCLOCK */ 1032 1033/* 1034 * Initialize a new timecounter and possibly use it. 1035 */ 1036void 1037tc_init(struct timecounter *tc) 1038{ 1039 u_int u; 1040 struct sysctl_oid *tc_root; 1041 1042 u = tc->tc_frequency / tc->tc_counter_mask; 1043 /* XXX: We need some margin here, 10% is a guess */ 1044 u *= 11; 1045 u /= 10; 1046 if (u > hz && tc->tc_quality >= 0) { 1047 tc->tc_quality = -2000; 1048 if (bootverbose) { 1049 printf("Timecounter \"%s\" frequency %ju Hz", 1050 tc->tc_name, (uintmax_t)tc->tc_frequency); 1051 printf(" -- Insufficient hz, needs at least %u\n", u); 1052 } 1053 } else if (tc->tc_quality >= 0 || bootverbose) { 1054 printf("Timecounter \"%s\" frequency %ju Hz quality %d\n", 1055 tc->tc_name, (uintmax_t)tc->tc_frequency, 1056 tc->tc_quality); 1057 } 1058 1059 tc->tc_next = timecounters; 1060 timecounters = tc; 1061 /* 1062 * Set up sysctl tree for this counter. 1063 */ 1064 tc_root = SYSCTL_ADD_NODE(NULL, 1065 SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name, 1066 CTLFLAG_RW, 0, "timecounter description"); 1067 SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1068 "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0, 1069 "mask for implemented bits"); 1070 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1071 "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc), 1072 sysctl_kern_timecounter_get, "IU", "current timecounter value"); 1073 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1074 "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc), 1075 sysctl_kern_timecounter_freq, "QU", "timecounter frequency"); 1076 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1077 "quality", CTLFLAG_RD, &(tc->tc_quality), 0, 1078 "goodness of time counter"); 1079 /* 1080 * Never automatically use a timecounter with negative quality. 1081 * Even though we run on the dummy counter, switching here may be 1082 * worse since this timecounter may not be monotonous. 1083 */ 1084 if (tc->tc_quality < 0) 1085 return; 1086 if (tc->tc_quality < timecounter->tc_quality) 1087 return; 1088 if (tc->tc_quality == timecounter->tc_quality && 1089 tc->tc_frequency < timecounter->tc_frequency) 1090 return; 1091 (void)tc->tc_get_timecount(tc); 1092 (void)tc->tc_get_timecount(tc); 1093 timecounter = tc; 1094} 1095 1096/* Report the frequency of the current timecounter. */ 1097uint64_t 1098tc_getfrequency(void) 1099{ 1100 1101 return (timehands->th_counter->tc_frequency); 1102} 1103 1104/* 1105 * Step our concept of UTC. This is done by modifying our estimate of 1106 * when we booted. 1107 * XXX: not locked. 1108 */ 1109void 1110tc_setclock(struct timespec *ts) 1111{ 1112 struct timespec tbef, taft; 1113 struct bintime bt, bt2; 1114 1115 cpu_tick_calibrate(1); 1116 nanotime(&tbef); 1117 timespec2bintime(ts, &bt); 1118 binuptime(&bt2); 1119 bintime_sub(&bt, &bt2); 1120 bintime_add(&bt2, &boottimebin); 1121 boottimebin = bt; 1122 bintime2timeval(&bt, &boottime); 1123 1124 /* XXX fiddle all the little crinkly bits around the fiords... */ 1125 tc_windup(); 1126 nanotime(&taft); 1127 if (timestepwarnings) { 1128 log(LOG_INFO, 1129 "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n", 1130 (intmax_t)tbef.tv_sec, tbef.tv_nsec, 1131 (intmax_t)taft.tv_sec, taft.tv_nsec, 1132 (intmax_t)ts->tv_sec, ts->tv_nsec); 1133 } 1134 cpu_tick_calibrate(1); 1135} 1136 1137/* 1138 * Initialize the next struct timehands in the ring and make 1139 * it the active timehands. Along the way we might switch to a different 1140 * timecounter and/or do seconds processing in NTP. Slightly magic. 1141 */ 1142static void 1143tc_windup(void) 1144{ 1145 struct bintime bt; 1146 struct timehands *th, *tho; 1147 uint64_t scale; 1148 u_int delta, ncount, ogen; 1149 int i; 1150 time_t t; 1151 1152 /* 1153 * Make the next timehands a copy of the current one, but do not 1154 * overwrite the generation or next pointer. While we update 1155 * the contents, the generation must be zero. 1156 */ 1157 tho = timehands; 1158 th = tho->th_next; 1159 ogen = th->th_generation; 1160 th->th_generation = 0; 1161 bcopy(tho, th, offsetof(struct timehands, th_generation)); 1162 1163 /* 1164 * Capture a timecounter delta on the current timecounter and if 1165 * changing timecounters, a counter value from the new timecounter. 1166 * Update the offset fields accordingly. 1167 */ 1168 delta = tc_delta(th); 1169 if (th->th_counter != timecounter) 1170 ncount = timecounter->tc_get_timecount(timecounter); 1171 else 1172 ncount = 0; 1173#ifdef FFCLOCK 1174 ffclock_windup(delta); 1175#endif 1176 th->th_offset_count += delta; 1177 th->th_offset_count &= th->th_counter->tc_counter_mask; 1178 while (delta > th->th_counter->tc_frequency) { 1179 /* Eat complete unadjusted seconds. */ 1180 delta -= th->th_counter->tc_frequency; 1181 th->th_offset.sec++; 1182 } 1183 if ((delta > th->th_counter->tc_frequency / 2) && 1184 (th->th_scale * delta < ((uint64_t)1 << 63))) { 1185 /* The product th_scale * delta just barely overflows. */ 1186 th->th_offset.sec++; 1187 } 1188 bintime_addx(&th->th_offset, th->th_scale * delta); 1189 1190 /* 1191 * Hardware latching timecounters may not generate interrupts on 1192 * PPS events, so instead we poll them. There is a finite risk that 1193 * the hardware might capture a count which is later than the one we 1194 * got above, and therefore possibly in the next NTP second which might 1195 * have a different rate than the current NTP second. It doesn't 1196 * matter in practice. 1197 */ 1198 if (tho->th_counter->tc_poll_pps) 1199 tho->th_counter->tc_poll_pps(tho->th_counter); 1200 1201 /* 1202 * Deal with NTP second processing. The for loop normally 1203 * iterates at most once, but in extreme situations it might 1204 * keep NTP sane if timeouts are not run for several seconds. 1205 * At boot, the time step can be large when the TOD hardware 1206 * has been read, so on really large steps, we call 1207 * ntp_update_second only twice. We need to call it twice in 1208 * case we missed a leap second. 1209 */ 1210 bt = th->th_offset; 1211 bintime_add(&bt, &boottimebin); 1212 i = bt.sec - tho->th_microtime.tv_sec; 1213 if (i > LARGE_STEP) 1214 i = 2; 1215 for (; i > 0; i--) { 1216 t = bt.sec; 1217 ntp_update_second(&th->th_adjustment, &bt.sec); 1218 if (bt.sec != t) 1219 boottimebin.sec += bt.sec - t; 1220 } 1221 /* Update the UTC timestamps used by the get*() functions. */ 1222 /* XXX shouldn't do this here. Should force non-`get' versions. */ 1223 bintime2timeval(&bt, &th->th_microtime); 1224 bintime2timespec(&bt, &th->th_nanotime); 1225 1226 /* Now is a good time to change timecounters. */ 1227 if (th->th_counter != timecounter) { 1228#ifndef __arm__ 1229 if ((timecounter->tc_flags & TC_FLAGS_C3STOP) != 0) 1230 cpu_disable_deep_sleep++; 1231 if ((th->th_counter->tc_flags & TC_FLAGS_C3STOP) != 0) 1232 cpu_disable_deep_sleep--; 1233#endif 1234 th->th_counter = timecounter; 1235 th->th_offset_count = ncount; 1236 tc_min_ticktock_freq = max(1, timecounter->tc_frequency / 1237 (((uint64_t)timecounter->tc_counter_mask + 1) / 3)); 1238#ifdef FFCLOCK 1239 ffclock_change_tc(th); 1240#endif 1241 } 1242 1243 /*- 1244 * Recalculate the scaling factor. We want the number of 1/2^64 1245 * fractions of a second per period of the hardware counter, taking 1246 * into account the th_adjustment factor which the NTP PLL/adjtime(2) 1247 * processing provides us with. 1248 * 1249 * The th_adjustment is nanoseconds per second with 32 bit binary 1250 * fraction and we want 64 bit binary fraction of second: 1251 * 1252 * x = a * 2^32 / 10^9 = a * 4.294967296 1253 * 1254 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int 1255 * we can only multiply by about 850 without overflowing, that 1256 * leaves no suitably precise fractions for multiply before divide. 1257 * 1258 * Divide before multiply with a fraction of 2199/512 results in a 1259 * systematic undercompensation of 10PPM of th_adjustment. On a 1260 * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. 1261 * 1262 * We happily sacrifice the lowest of the 64 bits of our result 1263 * to the goddess of code clarity. 1264 * 1265 */ 1266 scale = (uint64_t)1 << 63; 1267 scale += (th->th_adjustment / 1024) * 2199; 1268 scale /= th->th_counter->tc_frequency; 1269 th->th_scale = scale * 2; 1270 1271#ifdef FFCLOCK 1272 if (sysclock_active != sysclock.active) 1273 change_sysclock(sysclock_active); 1274#endif 1275 1276 /* 1277 * Now that the struct timehands is again consistent, set the new 1278 * generation number, making sure to not make it zero. 1279 */ 1280 if (++ogen == 0) 1281 ogen = 1; 1282 th->th_generation = ogen; 1283 1284 /* Go live with the new struct timehands. */ 1285#ifdef FFCLOCK 1286 switch (sysclock_active) { 1287 case SYSCLOCK_FBCK: 1288#endif 1289 time_second = th->th_microtime.tv_sec; 1290 time_uptime = th->th_offset.sec; 1291#ifdef FFCLOCK 1292 break; 1293 case SYSCLOCK_FFWD: 1294 time_second = fftimehands->tick_time_lerp.sec; 1295 time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec; 1296 break; 1297 } 1298#endif 1299 1300 timehands = th; 1301} 1302 1303/* Report or change the active timecounter hardware. */ 1304static int 1305sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) 1306{ 1307 char newname[32]; 1308 struct timecounter *newtc, *tc; 1309 int error; 1310 1311 tc = timecounter; 1312 strlcpy(newname, tc->tc_name, sizeof(newname)); 1313 1314 error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req); 1315 if (error != 0 || req->newptr == NULL || 1316 strcmp(newname, tc->tc_name) == 0) 1317 return (error); 1318 for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) { 1319 if (strcmp(newname, newtc->tc_name) != 0) 1320 continue; 1321 1322 /* Warm up new timecounter. */ 1323 (void)newtc->tc_get_timecount(newtc); 1324 (void)newtc->tc_get_timecount(newtc); 1325 1326 timecounter = newtc; 1327 return (0); 1328 } 1329 return (EINVAL); 1330} 1331 1332SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW, 1333 0, 0, sysctl_kern_timecounter_hardware, "A", 1334 "Timecounter hardware selected"); 1335 1336 1337/* Report or change the active timecounter hardware. */ 1338static int 1339sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS) 1340{ 1341 char buf[32], *spc; 1342 struct timecounter *tc; 1343 int error; 1344 1345 spc = ""; 1346 error = 0; 1347 for (tc = timecounters; error == 0 && tc != NULL; tc = tc->tc_next) { 1348 sprintf(buf, "%s%s(%d)", 1349 spc, tc->tc_name, tc->tc_quality); 1350 error = SYSCTL_OUT(req, buf, strlen(buf)); 1351 spc = " "; 1352 } 1353 return (error); 1354} 1355 1356SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD, 1357 0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected"); 1358 1359/* 1360 * RFC 2783 PPS-API implementation. 1361 */ 1362 1363int 1364pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps) 1365{ 1366 pps_params_t *app; 1367 struct pps_fetch_args *fapi; 1368#ifdef PPS_SYNC 1369 struct pps_kcbind_args *kapi; 1370#endif 1371 1372 KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl")); 1373 switch (cmd) { 1374 case PPS_IOC_CREATE: 1375 return (0); 1376 case PPS_IOC_DESTROY: 1377 return (0); 1378 case PPS_IOC_SETPARAMS: 1379 app = (pps_params_t *)data; 1380 if (app->mode & ~pps->ppscap) 1381 return (EINVAL); 1382 pps->ppsparam = *app; 1383 return (0); 1384 case PPS_IOC_GETPARAMS: 1385 app = (pps_params_t *)data; 1386 *app = pps->ppsparam; 1387 app->api_version = PPS_API_VERS_1; 1388 return (0); 1389 case PPS_IOC_GETCAP: 1390 *(int*)data = pps->ppscap; 1391 return (0); 1392 case PPS_IOC_FETCH: 1393 fapi = (struct pps_fetch_args *)data; 1394 if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC) 1395 return (EINVAL); 1396 if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) 1397 return (EOPNOTSUPP); 1398 pps->ppsinfo.current_mode = pps->ppsparam.mode; 1399 fapi->pps_info_buf = pps->ppsinfo; 1400 return (0); 1401 case PPS_IOC_KCBIND: 1402#ifdef PPS_SYNC 1403 kapi = (struct pps_kcbind_args *)data; 1404 /* XXX Only root should be able to do this */ 1405 if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC) 1406 return (EINVAL); 1407 if (kapi->kernel_consumer != PPS_KC_HARDPPS) 1408 return (EINVAL); 1409 if (kapi->edge & ~pps->ppscap) 1410 return (EINVAL); 1411 pps->kcmode = kapi->edge; 1412 return (0); 1413#else 1414 return (EOPNOTSUPP); 1415#endif 1416 default: 1417 return (ENOIOCTL); 1418 } 1419} 1420 1421void 1422pps_init(struct pps_state *pps) 1423{ 1424 pps->ppscap |= PPS_TSFMT_TSPEC; 1425 if (pps->ppscap & PPS_CAPTUREASSERT) 1426 pps->ppscap |= PPS_OFFSETASSERT; 1427 if (pps->ppscap & PPS_CAPTURECLEAR) 1428 pps->ppscap |= PPS_OFFSETCLEAR; 1429} 1430 1431void 1432pps_capture(struct pps_state *pps) 1433{ 1434 struct timehands *th; 1435 1436 KASSERT(pps != NULL, ("NULL pps pointer in pps_capture")); 1437 th = timehands; 1438 pps->capgen = th->th_generation; 1439 pps->capth = th; 1440 pps->capcount = th->th_counter->tc_get_timecount(th->th_counter); 1441 if (pps->capgen != th->th_generation) 1442 pps->capgen = 0; 1443} 1444 1445void 1446pps_event(struct pps_state *pps, int event) 1447{ 1448 struct bintime bt; 1449 struct timespec ts, *tsp, *osp; 1450 u_int tcount, *pcount; 1451 int foff, fhard; 1452 pps_seq_t *pseq; 1453 1454 KASSERT(pps != NULL, ("NULL pps pointer in pps_event")); 1455 /* If the timecounter was wound up underneath us, bail out. */ 1456 if (pps->capgen == 0 || pps->capgen != pps->capth->th_generation) 1457 return; 1458 1459 /* Things would be easier with arrays. */ 1460 if (event == PPS_CAPTUREASSERT) { 1461 tsp = &pps->ppsinfo.assert_timestamp; 1462 osp = &pps->ppsparam.assert_offset; 1463 foff = pps->ppsparam.mode & PPS_OFFSETASSERT; 1464 fhard = pps->kcmode & PPS_CAPTUREASSERT; 1465 pcount = &pps->ppscount[0]; 1466 pseq = &pps->ppsinfo.assert_sequence; 1467 } else { 1468 tsp = &pps->ppsinfo.clear_timestamp; 1469 osp = &pps->ppsparam.clear_offset; 1470 foff = pps->ppsparam.mode & PPS_OFFSETCLEAR; 1471 fhard = pps->kcmode & PPS_CAPTURECLEAR; 1472 pcount = &pps->ppscount[1]; 1473 pseq = &pps->ppsinfo.clear_sequence; 1474 } 1475 1476 /* 1477 * If the timecounter changed, we cannot compare the count values, so 1478 * we have to drop the rest of the PPS-stuff until the next event. 1479 */ 1480 if (pps->ppstc != pps->capth->th_counter) { 1481 pps->ppstc = pps->capth->th_counter; 1482 *pcount = pps->capcount; 1483 pps->ppscount[2] = pps->capcount; 1484 return; 1485 } 1486 1487 /* Convert the count to a timespec. */ 1488 tcount = pps->capcount - pps->capth->th_offset_count; 1489 tcount &= pps->capth->th_counter->tc_counter_mask; 1490 bt = pps->capth->th_offset; 1491 bintime_addx(&bt, pps->capth->th_scale * tcount); 1492 bintime_add(&bt, &boottimebin); 1493 bintime2timespec(&bt, &ts); 1494 1495 /* If the timecounter was wound up underneath us, bail out. */ 1496 if (pps->capgen != pps->capth->th_generation) 1497 return; 1498 1499 *pcount = pps->capcount; 1500 (*pseq)++; 1501 *tsp = ts; 1502 1503 if (foff) { 1504 timespecadd(tsp, osp); 1505 if (tsp->tv_nsec < 0) { 1506 tsp->tv_nsec += 1000000000; 1507 tsp->tv_sec -= 1; 1508 } 1509 } 1510#ifdef PPS_SYNC 1511 if (fhard) { 1512 uint64_t scale; 1513 1514 /* 1515 * Feed the NTP PLL/FLL. 1516 * The FLL wants to know how many (hardware) nanoseconds 1517 * elapsed since the previous event. 1518 */ 1519 tcount = pps->capcount - pps->ppscount[2]; 1520 pps->ppscount[2] = pps->capcount; 1521 tcount &= pps->capth->th_counter->tc_counter_mask; 1522 scale = (uint64_t)1 << 63; 1523 scale /= pps->capth->th_counter->tc_frequency; 1524 scale *= 2; 1525 bt.sec = 0; 1526 bt.frac = 0; 1527 bintime_addx(&bt, scale * tcount); 1528 bintime2timespec(&bt, &ts); 1529 hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec); 1530 } 1531#endif 1532} 1533 1534/* 1535 * Timecounters need to be updated every so often to prevent the hardware 1536 * counter from overflowing. Updating also recalculates the cached values 1537 * used by the get*() family of functions, so their precision depends on 1538 * the update frequency. 1539 */ 1540 1541static int tc_tick; 1542SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0, 1543 "Approximate number of hardclock ticks in a millisecond"); 1544 1545void 1546tc_ticktock(int cnt) 1547{ 1548 static int count; 1549 1550 count += cnt; 1551 if (count < tc_tick) 1552 return; 1553 count = 0; 1554 tc_windup(); 1555} 1556 1557static void 1558inittimecounter(void *dummy) 1559{ 1560 u_int p; 1561 1562 /* 1563 * Set the initial timeout to 1564 * max(1, <approx. number of hardclock ticks in a millisecond>). 1565 * People should probably not use the sysctl to set the timeout 1566 * to smaller than its inital value, since that value is the 1567 * smallest reasonable one. If they want better timestamps they 1568 * should use the non-"get"* functions. 1569 */ 1570 if (hz > 1000) 1571 tc_tick = (hz + 500) / 1000; 1572 else 1573 tc_tick = 1; 1574 p = (tc_tick * 1000000) / hz; 1575 printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000); 1576 1577#ifdef FFCLOCK 1578 ffclock_init(); 1579 change_sysclock(sysclock_active); 1580#endif 1581 /* warm up new timecounter (again) and get rolling. */ 1582 (void)timecounter->tc_get_timecount(timecounter); 1583 (void)timecounter->tc_get_timecount(timecounter); 1584 tc_windup(); 1585} 1586 1587SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL); 1588 1589/* Cpu tick handling -------------------------------------------------*/ 1590 1591static int cpu_tick_variable; 1592static uint64_t cpu_tick_frequency; 1593 1594static uint64_t 1595tc_cpu_ticks(void) 1596{ 1597 static uint64_t base; 1598 static unsigned last; 1599 unsigned u; 1600 struct timecounter *tc; 1601 1602 tc = timehands->th_counter; 1603 u = tc->tc_get_timecount(tc) & tc->tc_counter_mask; 1604 if (u < last) 1605 base += (uint64_t)tc->tc_counter_mask + 1; 1606 last = u; 1607 return (u + base); 1608} 1609 1610void 1611cpu_tick_calibration(void) 1612{ 1613 static time_t last_calib; 1614 1615 if (time_uptime != last_calib && !(time_uptime & 0xf)) { 1616 cpu_tick_calibrate(0); 1617 last_calib = time_uptime; 1618 } 1619} 1620 1621/* 1622 * This function gets called every 16 seconds on only one designated 1623 * CPU in the system from hardclock() via cpu_tick_calibration()(). 1624 * 1625 * Whenever the real time clock is stepped we get called with reset=1 1626 * to make sure we handle suspend/resume and similar events correctly. 1627 */ 1628 1629static void 1630cpu_tick_calibrate(int reset) 1631{ 1632 static uint64_t c_last; 1633 uint64_t c_this, c_delta; 1634 static struct bintime t_last; 1635 struct bintime t_this, t_delta; 1636 uint32_t divi; 1637 1638 if (reset) { 1639 /* The clock was stepped, abort & reset */ 1640 t_last.sec = 0; 1641 return; 1642 } 1643 1644 /* we don't calibrate fixed rate cputicks */ 1645 if (!cpu_tick_variable) 1646 return; 1647 1648 getbinuptime(&t_this); 1649 c_this = cpu_ticks(); 1650 if (t_last.sec != 0) { 1651 c_delta = c_this - c_last; 1652 t_delta = t_this; 1653 bintime_sub(&t_delta, &t_last); 1654 /* 1655 * Headroom: 1656 * 2^(64-20) / 16[s] = 1657 * 2^(44) / 16[s] = 1658 * 17.592.186.044.416 / 16 = 1659 * 1.099.511.627.776 [Hz] 1660 */ 1661 divi = t_delta.sec << 20; 1662 divi |= t_delta.frac >> (64 - 20); 1663 c_delta <<= 20; 1664 c_delta /= divi; 1665 if (c_delta > cpu_tick_frequency) { 1666 if (0 && bootverbose) 1667 printf("cpu_tick increased to %ju Hz\n", 1668 c_delta); 1669 cpu_tick_frequency = c_delta; 1670 } 1671 } 1672 c_last = c_this; 1673 t_last = t_this; 1674} 1675 1676void 1677set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var) 1678{ 1679 1680 if (func == NULL) { 1681 cpu_ticks = tc_cpu_ticks; 1682 } else { 1683 cpu_tick_frequency = freq; 1684 cpu_tick_variable = var; 1685 cpu_ticks = func; 1686 } 1687} 1688 1689uint64_t 1690cpu_tickrate(void) 1691{ 1692 1693 if (cpu_ticks == tc_cpu_ticks) 1694 return (tc_getfrequency()); 1695 return (cpu_tick_frequency); 1696} 1697 1698/* 1699 * We need to be slightly careful converting cputicks to microseconds. 1700 * There is plenty of margin in 64 bits of microseconds (half a million 1701 * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply 1702 * before divide conversion (to retain precision) we find that the 1703 * margin shrinks to 1.5 hours (one millionth of 146y). 1704 * With a three prong approach we never lose significant bits, no 1705 * matter what the cputick rate and length of timeinterval is. 1706 */ 1707 1708uint64_t 1709cputick2usec(uint64_t tick) 1710{ 1711 1712 if (tick > 18446744073709551LL) /* floor(2^64 / 1000) */ 1713 return (tick / (cpu_tickrate() / 1000000LL)); 1714 else if (tick > 18446744073709LL) /* floor(2^64 / 1000000) */ 1715 return ((tick * 1000LL) / (cpu_tickrate() / 1000LL)); 1716 else 1717 return ((tick * 1000000LL) / cpu_tickrate()); 1718} 1719 1720cpu_tick_f *cpu_ticks = tc_cpu_ticks; 1721