kern_tc.c revision 286423
1/*- 2 * ---------------------------------------------------------------------------- 3 * "THE BEER-WARE LICENSE" (Revision 42): 4 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you 5 * can do whatever you want with this stuff. If we meet some day, and you think 6 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 7 * ---------------------------------------------------------------------------- 8 * 9 * Copyright (c) 2011 The FreeBSD Foundation 10 * All rights reserved. 11 * 12 * Portions of this software were developed by Julien Ridoux at the University 13 * of Melbourne under sponsorship from the FreeBSD Foundation. 14 */ 15 16#include <sys/cdefs.h> 17__FBSDID("$FreeBSD: head/sys/kern/kern_tc.c 286423 2015-08-07 21:14:19Z ian $"); 18 19#include "opt_compat.h" 20#include "opt_ntp.h" 21#include "opt_ffclock.h" 22 23#include <sys/param.h> 24#include <sys/kernel.h> 25#include <sys/limits.h> 26#include <sys/lock.h> 27#include <sys/mutex.h> 28#include <sys/sbuf.h> 29#include <sys/sysctl.h> 30#include <sys/syslog.h> 31#include <sys/systm.h> 32#include <sys/timeffc.h> 33#include <sys/timepps.h> 34#include <sys/timetc.h> 35#include <sys/timex.h> 36#include <sys/vdso.h> 37 38/* 39 * A large step happens on boot. This constant detects such steps. 40 * It is relatively small so that ntp_update_second gets called enough 41 * in the typical 'missed a couple of seconds' case, but doesn't loop 42 * forever when the time step is large. 43 */ 44#define LARGE_STEP 200 45 46/* 47 * Implement a dummy timecounter which we can use until we get a real one 48 * in the air. This allows the console and other early stuff to use 49 * time services. 50 */ 51 52static u_int 53dummy_get_timecount(struct timecounter *tc) 54{ 55 static u_int now; 56 57 return (++now); 58} 59 60static struct timecounter dummy_timecounter = { 61 dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000 62}; 63 64struct timehands { 65 /* These fields must be initialized by the driver. */ 66 struct timecounter *th_counter; 67 int64_t th_adjustment; 68 uint64_t th_scale; 69 u_int th_offset_count; 70 struct bintime th_offset; 71 struct timeval th_microtime; 72 struct timespec th_nanotime; 73 /* Fields not to be copied in tc_windup start with th_generation. */ 74 u_int th_generation; 75 struct timehands *th_next; 76}; 77 78static struct timehands th0; 79static struct timehands th9 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th0}; 80static struct timehands th8 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th9}; 81static struct timehands th7 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th8}; 82static struct timehands th6 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th7}; 83static struct timehands th5 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th6}; 84static struct timehands th4 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th5}; 85static struct timehands th3 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th4}; 86static struct timehands th2 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th3}; 87static struct timehands th1 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th2}; 88static struct timehands th0 = { 89 &dummy_timecounter, 90 0, 91 (uint64_t)-1 / 1000000, 92 0, 93 {1, 0}, 94 {0, 0}, 95 {0, 0}, 96 1, 97 &th1 98}; 99 100static struct timehands *volatile timehands = &th0; 101struct timecounter *timecounter = &dummy_timecounter; 102static struct timecounter *timecounters = &dummy_timecounter; 103 104int tc_min_ticktock_freq = 1; 105 106volatile time_t time_second = 1; 107volatile time_t time_uptime = 1; 108 109struct bintime boottimebin; 110struct timeval boottime; 111static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS); 112SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD, 113 NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime"); 114 115SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, ""); 116static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, ""); 117 118static int timestepwarnings; 119SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW, 120 ×tepwarnings, 0, "Log time steps"); 121 122struct bintime bt_timethreshold; 123struct bintime bt_tickthreshold; 124sbintime_t sbt_timethreshold; 125sbintime_t sbt_tickthreshold; 126struct bintime tc_tick_bt; 127sbintime_t tc_tick_sbt; 128int tc_precexp; 129int tc_timepercentage = TC_DEFAULTPERC; 130static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS); 131SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation, 132 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 0, 0, 133 sysctl_kern_timecounter_adjprecision, "I", 134 "Allowed time interval deviation in percents"); 135 136static void tc_windup(void); 137static void cpu_tick_calibrate(int); 138 139void dtrace_getnanotime(struct timespec *tsp); 140 141static int 142sysctl_kern_boottime(SYSCTL_HANDLER_ARGS) 143{ 144#ifndef __mips__ 145#ifdef SCTL_MASK32 146 int tv[2]; 147 148 if (req->flags & SCTL_MASK32) { 149 tv[0] = boottime.tv_sec; 150 tv[1] = boottime.tv_usec; 151 return SYSCTL_OUT(req, tv, sizeof(tv)); 152 } else 153#endif 154#endif 155 return SYSCTL_OUT(req, &boottime, sizeof(boottime)); 156} 157 158static int 159sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS) 160{ 161 u_int ncount; 162 struct timecounter *tc = arg1; 163 164 ncount = tc->tc_get_timecount(tc); 165 return sysctl_handle_int(oidp, &ncount, 0, req); 166} 167 168static int 169sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS) 170{ 171 uint64_t freq; 172 struct timecounter *tc = arg1; 173 174 freq = tc->tc_frequency; 175 return sysctl_handle_64(oidp, &freq, 0, req); 176} 177 178/* 179 * Return the difference between the timehands' counter value now and what 180 * was when we copied it to the timehands' offset_count. 181 */ 182static __inline u_int 183tc_delta(struct timehands *th) 184{ 185 struct timecounter *tc; 186 187 tc = th->th_counter; 188 return ((tc->tc_get_timecount(tc) - th->th_offset_count) & 189 tc->tc_counter_mask); 190} 191 192/* 193 * Functions for reading the time. We have to loop until we are sure that 194 * the timehands that we operated on was not updated under our feet. See 195 * the comment in <sys/time.h> for a description of these 12 functions. 196 */ 197 198#ifdef FFCLOCK 199void 200fbclock_binuptime(struct bintime *bt) 201{ 202 struct timehands *th; 203 unsigned int gen; 204 205 do { 206 th = timehands; 207 gen = atomic_load_acq_int(&th->th_generation); 208 *bt = th->th_offset; 209 bintime_addx(bt, th->th_scale * tc_delta(th)); 210 atomic_thread_fence_acq(); 211 } while (gen == 0 || gen != th->th_generation); 212} 213 214void 215fbclock_nanouptime(struct timespec *tsp) 216{ 217 struct bintime bt; 218 219 fbclock_binuptime(&bt); 220 bintime2timespec(&bt, tsp); 221} 222 223void 224fbclock_microuptime(struct timeval *tvp) 225{ 226 struct bintime bt; 227 228 fbclock_binuptime(&bt); 229 bintime2timeval(&bt, tvp); 230} 231 232void 233fbclock_bintime(struct bintime *bt) 234{ 235 236 fbclock_binuptime(bt); 237 bintime_add(bt, &boottimebin); 238} 239 240void 241fbclock_nanotime(struct timespec *tsp) 242{ 243 struct bintime bt; 244 245 fbclock_bintime(&bt); 246 bintime2timespec(&bt, tsp); 247} 248 249void 250fbclock_microtime(struct timeval *tvp) 251{ 252 struct bintime bt; 253 254 fbclock_bintime(&bt); 255 bintime2timeval(&bt, tvp); 256} 257 258void 259fbclock_getbinuptime(struct bintime *bt) 260{ 261 struct timehands *th; 262 unsigned int gen; 263 264 do { 265 th = timehands; 266 gen = atomic_load_acq_int(&th->th_generation); 267 *bt = th->th_offset; 268 atomic_thread_fence_acq(); 269 } while (gen == 0 || gen != th->th_generation); 270} 271 272void 273fbclock_getnanouptime(struct timespec *tsp) 274{ 275 struct timehands *th; 276 unsigned int gen; 277 278 do { 279 th = timehands; 280 gen = atomic_load_acq_int(&th->th_generation); 281 bintime2timespec(&th->th_offset, tsp); 282 atomic_thread_fence_acq(); 283 } while (gen == 0 || gen != th->th_generation); 284} 285 286void 287fbclock_getmicrouptime(struct timeval *tvp) 288{ 289 struct timehands *th; 290 unsigned int gen; 291 292 do { 293 th = timehands; 294 gen = atomic_load_acq_int(&th->th_generation); 295 bintime2timeval(&th->th_offset, tvp); 296 atomic_thread_fence_acq(); 297 } while (gen == 0 || gen != th->th_generation); 298} 299 300void 301fbclock_getbintime(struct bintime *bt) 302{ 303 struct timehands *th; 304 unsigned int gen; 305 306 do { 307 th = timehands; 308 gen = atomic_load_acq_int(&th->th_generation); 309 *bt = th->th_offset; 310 atomic_thread_fence_acq(); 311 } while (gen == 0 || gen != th->th_generation); 312 bintime_add(bt, &boottimebin); 313} 314 315void 316fbclock_getnanotime(struct timespec *tsp) 317{ 318 struct timehands *th; 319 unsigned int gen; 320 321 do { 322 th = timehands; 323 gen = atomic_load_acq_int(&th->th_generation); 324 *tsp = th->th_nanotime; 325 atomic_thread_fence_acq(); 326 } while (gen == 0 || gen != th->th_generation); 327} 328 329void 330fbclock_getmicrotime(struct timeval *tvp) 331{ 332 struct timehands *th; 333 unsigned int gen; 334 335 do { 336 th = timehands; 337 gen = atomic_load_acq_int(&th->th_generation); 338 *tvp = th->th_microtime; 339 atomic_thread_fence_acq(); 340 } while (gen == 0 || gen != th->th_generation); 341} 342#else /* !FFCLOCK */ 343void 344binuptime(struct bintime *bt) 345{ 346 struct timehands *th; 347 u_int gen; 348 349 do { 350 th = timehands; 351 gen = atomic_load_acq_int(&th->th_generation); 352 *bt = th->th_offset; 353 bintime_addx(bt, th->th_scale * tc_delta(th)); 354 atomic_thread_fence_acq(); 355 } while (gen == 0 || gen != th->th_generation); 356} 357 358void 359nanouptime(struct timespec *tsp) 360{ 361 struct bintime bt; 362 363 binuptime(&bt); 364 bintime2timespec(&bt, tsp); 365} 366 367void 368microuptime(struct timeval *tvp) 369{ 370 struct bintime bt; 371 372 binuptime(&bt); 373 bintime2timeval(&bt, tvp); 374} 375 376void 377bintime(struct bintime *bt) 378{ 379 380 binuptime(bt); 381 bintime_add(bt, &boottimebin); 382} 383 384void 385nanotime(struct timespec *tsp) 386{ 387 struct bintime bt; 388 389 bintime(&bt); 390 bintime2timespec(&bt, tsp); 391} 392 393void 394microtime(struct timeval *tvp) 395{ 396 struct bintime bt; 397 398 bintime(&bt); 399 bintime2timeval(&bt, tvp); 400} 401 402void 403getbinuptime(struct bintime *bt) 404{ 405 struct timehands *th; 406 u_int gen; 407 408 do { 409 th = timehands; 410 gen = atomic_load_acq_int(&th->th_generation); 411 *bt = th->th_offset; 412 atomic_thread_fence_acq(); 413 } while (gen == 0 || gen != th->th_generation); 414} 415 416void 417getnanouptime(struct timespec *tsp) 418{ 419 struct timehands *th; 420 u_int gen; 421 422 do { 423 th = timehands; 424 gen = atomic_load_acq_int(&th->th_generation); 425 bintime2timespec(&th->th_offset, tsp); 426 atomic_thread_fence_acq(); 427 } while (gen == 0 || gen != th->th_generation); 428} 429 430void 431getmicrouptime(struct timeval *tvp) 432{ 433 struct timehands *th; 434 u_int gen; 435 436 do { 437 th = timehands; 438 gen = atomic_load_acq_int(&th->th_generation); 439 bintime2timeval(&th->th_offset, tvp); 440 atomic_thread_fence_acq(); 441 } while (gen == 0 || gen != th->th_generation); 442} 443 444void 445getbintime(struct bintime *bt) 446{ 447 struct timehands *th; 448 u_int gen; 449 450 do { 451 th = timehands; 452 gen = atomic_load_acq_int(&th->th_generation); 453 *bt = th->th_offset; 454 atomic_thread_fence_acq(); 455 } while (gen == 0 || gen != th->th_generation); 456 bintime_add(bt, &boottimebin); 457} 458 459void 460getnanotime(struct timespec *tsp) 461{ 462 struct timehands *th; 463 u_int gen; 464 465 do { 466 th = timehands; 467 gen = atomic_load_acq_int(&th->th_generation); 468 *tsp = th->th_nanotime; 469 atomic_thread_fence_acq(); 470 } while (gen == 0 || gen != th->th_generation); 471} 472 473void 474getmicrotime(struct timeval *tvp) 475{ 476 struct timehands *th; 477 u_int gen; 478 479 do { 480 th = timehands; 481 gen = atomic_load_acq_int(&th->th_generation); 482 *tvp = th->th_microtime; 483 atomic_thread_fence_acq(); 484 } while (gen == 0 || gen != th->th_generation); 485} 486#endif /* FFCLOCK */ 487 488#ifdef FFCLOCK 489/* 490 * Support for feed-forward synchronization algorithms. This is heavily inspired 491 * by the timehands mechanism but kept independent from it. *_windup() functions 492 * have some connection to avoid accessing the timecounter hardware more than 493 * necessary. 494 */ 495 496/* Feed-forward clock estimates kept updated by the synchronization daemon. */ 497struct ffclock_estimate ffclock_estimate; 498struct bintime ffclock_boottime; /* Feed-forward boot time estimate. */ 499uint32_t ffclock_status; /* Feed-forward clock status. */ 500int8_t ffclock_updated; /* New estimates are available. */ 501struct mtx ffclock_mtx; /* Mutex on ffclock_estimate. */ 502 503struct fftimehands { 504 struct ffclock_estimate cest; 505 struct bintime tick_time; 506 struct bintime tick_time_lerp; 507 ffcounter tick_ffcount; 508 uint64_t period_lerp; 509 volatile uint8_t gen; 510 struct fftimehands *next; 511}; 512 513#define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x)) 514 515static struct fftimehands ffth[10]; 516static struct fftimehands *volatile fftimehands = ffth; 517 518static void 519ffclock_init(void) 520{ 521 struct fftimehands *cur; 522 struct fftimehands *last; 523 524 memset(ffth, 0, sizeof(ffth)); 525 526 last = ffth + NUM_ELEMENTS(ffth) - 1; 527 for (cur = ffth; cur < last; cur++) 528 cur->next = cur + 1; 529 last->next = ffth; 530 531 ffclock_updated = 0; 532 ffclock_status = FFCLOCK_STA_UNSYNC; 533 mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF); 534} 535 536/* 537 * Reset the feed-forward clock estimates. Called from inittodr() to get things 538 * kick started and uses the timecounter nominal frequency as a first period 539 * estimate. Note: this function may be called several time just after boot. 540 * Note: this is the only function that sets the value of boot time for the 541 * monotonic (i.e. uptime) version of the feed-forward clock. 542 */ 543void 544ffclock_reset_clock(struct timespec *ts) 545{ 546 struct timecounter *tc; 547 struct ffclock_estimate cest; 548 549 tc = timehands->th_counter; 550 memset(&cest, 0, sizeof(struct ffclock_estimate)); 551 552 timespec2bintime(ts, &ffclock_boottime); 553 timespec2bintime(ts, &(cest.update_time)); 554 ffclock_read_counter(&cest.update_ffcount); 555 cest.leapsec_next = 0; 556 cest.period = ((1ULL << 63) / tc->tc_frequency) << 1; 557 cest.errb_abs = 0; 558 cest.errb_rate = 0; 559 cest.status = FFCLOCK_STA_UNSYNC; 560 cest.leapsec_total = 0; 561 cest.leapsec = 0; 562 563 mtx_lock(&ffclock_mtx); 564 bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate)); 565 ffclock_updated = INT8_MAX; 566 mtx_unlock(&ffclock_mtx); 567 568 printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name, 569 (unsigned long long)tc->tc_frequency, (long)ts->tv_sec, 570 (unsigned long)ts->tv_nsec); 571} 572 573/* 574 * Sub-routine to convert a time interval measured in RAW counter units to time 575 * in seconds stored in bintime format. 576 * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be 577 * larger than the max value of u_int (on 32 bit architecture). Loop to consume 578 * extra cycles. 579 */ 580static void 581ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt) 582{ 583 struct bintime bt2; 584 ffcounter delta, delta_max; 585 586 delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1; 587 bintime_clear(bt); 588 do { 589 if (ffdelta > delta_max) 590 delta = delta_max; 591 else 592 delta = ffdelta; 593 bt2.sec = 0; 594 bt2.frac = period; 595 bintime_mul(&bt2, (unsigned int)delta); 596 bintime_add(bt, &bt2); 597 ffdelta -= delta; 598 } while (ffdelta > 0); 599} 600 601/* 602 * Update the fftimehands. 603 * Push the tick ffcount and time(s) forward based on current clock estimate. 604 * The conversion from ffcounter to bintime relies on the difference clock 605 * principle, whose accuracy relies on computing small time intervals. If a new 606 * clock estimate has been passed by the synchronisation daemon, make it 607 * current, and compute the linear interpolation for monotonic time if needed. 608 */ 609static void 610ffclock_windup(unsigned int delta) 611{ 612 struct ffclock_estimate *cest; 613 struct fftimehands *ffth; 614 struct bintime bt, gap_lerp; 615 ffcounter ffdelta; 616 uint64_t frac; 617 unsigned int polling; 618 uint8_t forward_jump, ogen; 619 620 /* 621 * Pick the next timehand, copy current ffclock estimates and move tick 622 * times and counter forward. 623 */ 624 forward_jump = 0; 625 ffth = fftimehands->next; 626 ogen = ffth->gen; 627 ffth->gen = 0; 628 cest = &ffth->cest; 629 bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate)); 630 ffdelta = (ffcounter)delta; 631 ffth->period_lerp = fftimehands->period_lerp; 632 633 ffth->tick_time = fftimehands->tick_time; 634 ffclock_convert_delta(ffdelta, cest->period, &bt); 635 bintime_add(&ffth->tick_time, &bt); 636 637 ffth->tick_time_lerp = fftimehands->tick_time_lerp; 638 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt); 639 bintime_add(&ffth->tick_time_lerp, &bt); 640 641 ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta; 642 643 /* 644 * Assess the status of the clock, if the last update is too old, it is 645 * likely the synchronisation daemon is dead and the clock is free 646 * running. 647 */ 648 if (ffclock_updated == 0) { 649 ffdelta = ffth->tick_ffcount - cest->update_ffcount; 650 ffclock_convert_delta(ffdelta, cest->period, &bt); 651 if (bt.sec > 2 * FFCLOCK_SKM_SCALE) 652 ffclock_status |= FFCLOCK_STA_UNSYNC; 653 } 654 655 /* 656 * If available, grab updated clock estimates and make them current. 657 * Recompute time at this tick using the updated estimates. The clock 658 * estimates passed the feed-forward synchronisation daemon may result 659 * in time conversion that is not monotonically increasing (just after 660 * the update). time_lerp is a particular linear interpolation over the 661 * synchronisation algo polling period that ensures monotonicity for the 662 * clock ids requesting it. 663 */ 664 if (ffclock_updated > 0) { 665 bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate)); 666 ffdelta = ffth->tick_ffcount - cest->update_ffcount; 667 ffth->tick_time = cest->update_time; 668 ffclock_convert_delta(ffdelta, cest->period, &bt); 669 bintime_add(&ffth->tick_time, &bt); 670 671 /* ffclock_reset sets ffclock_updated to INT8_MAX */ 672 if (ffclock_updated == INT8_MAX) 673 ffth->tick_time_lerp = ffth->tick_time; 674 675 if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >)) 676 forward_jump = 1; 677 else 678 forward_jump = 0; 679 680 bintime_clear(&gap_lerp); 681 if (forward_jump) { 682 gap_lerp = ffth->tick_time; 683 bintime_sub(&gap_lerp, &ffth->tick_time_lerp); 684 } else { 685 gap_lerp = ffth->tick_time_lerp; 686 bintime_sub(&gap_lerp, &ffth->tick_time); 687 } 688 689 /* 690 * The reset from the RTC clock may be far from accurate, and 691 * reducing the gap between real time and interpolated time 692 * could take a very long time if the interpolated clock insists 693 * on strict monotonicity. The clock is reset under very strict 694 * conditions (kernel time is known to be wrong and 695 * synchronization daemon has been restarted recently. 696 * ffclock_boottime absorbs the jump to ensure boot time is 697 * correct and uptime functions stay consistent. 698 */ 699 if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) && 700 ((cest->status & FFCLOCK_STA_UNSYNC) == 0) && 701 ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) { 702 if (forward_jump) 703 bintime_add(&ffclock_boottime, &gap_lerp); 704 else 705 bintime_sub(&ffclock_boottime, &gap_lerp); 706 ffth->tick_time_lerp = ffth->tick_time; 707 bintime_clear(&gap_lerp); 708 } 709 710 ffclock_status = cest->status; 711 ffth->period_lerp = cest->period; 712 713 /* 714 * Compute corrected period used for the linear interpolation of 715 * time. The rate of linear interpolation is capped to 5000PPM 716 * (5ms/s). 717 */ 718 if (bintime_isset(&gap_lerp)) { 719 ffdelta = cest->update_ffcount; 720 ffdelta -= fftimehands->cest.update_ffcount; 721 ffclock_convert_delta(ffdelta, cest->period, &bt); 722 polling = bt.sec; 723 bt.sec = 0; 724 bt.frac = 5000000 * (uint64_t)18446744073LL; 725 bintime_mul(&bt, polling); 726 if (bintime_cmp(&gap_lerp, &bt, >)) 727 gap_lerp = bt; 728 729 /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */ 730 frac = 0; 731 if (gap_lerp.sec > 0) { 732 frac -= 1; 733 frac /= ffdelta / gap_lerp.sec; 734 } 735 frac += gap_lerp.frac / ffdelta; 736 737 if (forward_jump) 738 ffth->period_lerp += frac; 739 else 740 ffth->period_lerp -= frac; 741 } 742 743 ffclock_updated = 0; 744 } 745 if (++ogen == 0) 746 ogen = 1; 747 ffth->gen = ogen; 748 fftimehands = ffth; 749} 750 751/* 752 * Adjust the fftimehands when the timecounter is changed. Stating the obvious, 753 * the old and new hardware counter cannot be read simultaneously. tc_windup() 754 * does read the two counters 'back to back', but a few cycles are effectively 755 * lost, and not accumulated in tick_ffcount. This is a fairly radical 756 * operation for a feed-forward synchronization daemon, and it is its job to not 757 * pushing irrelevant data to the kernel. Because there is no locking here, 758 * simply force to ignore pending or next update to give daemon a chance to 759 * realize the counter has changed. 760 */ 761static void 762ffclock_change_tc(struct timehands *th) 763{ 764 struct fftimehands *ffth; 765 struct ffclock_estimate *cest; 766 struct timecounter *tc; 767 uint8_t ogen; 768 769 tc = th->th_counter; 770 ffth = fftimehands->next; 771 ogen = ffth->gen; 772 ffth->gen = 0; 773 774 cest = &ffth->cest; 775 bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate)); 776 cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1; 777 cest->errb_abs = 0; 778 cest->errb_rate = 0; 779 cest->status |= FFCLOCK_STA_UNSYNC; 780 781 ffth->tick_ffcount = fftimehands->tick_ffcount; 782 ffth->tick_time_lerp = fftimehands->tick_time_lerp; 783 ffth->tick_time = fftimehands->tick_time; 784 ffth->period_lerp = cest->period; 785 786 /* Do not lock but ignore next update from synchronization daemon. */ 787 ffclock_updated--; 788 789 if (++ogen == 0) 790 ogen = 1; 791 ffth->gen = ogen; 792 fftimehands = ffth; 793} 794 795/* 796 * Retrieve feed-forward counter and time of last kernel tick. 797 */ 798void 799ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags) 800{ 801 struct fftimehands *ffth; 802 uint8_t gen; 803 804 /* 805 * No locking but check generation has not changed. Also need to make 806 * sure ffdelta is positive, i.e. ffcount > tick_ffcount. 807 */ 808 do { 809 ffth = fftimehands; 810 gen = ffth->gen; 811 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) 812 *bt = ffth->tick_time_lerp; 813 else 814 *bt = ffth->tick_time; 815 *ffcount = ffth->tick_ffcount; 816 } while (gen == 0 || gen != ffth->gen); 817} 818 819/* 820 * Absolute clock conversion. Low level function to convert ffcounter to 821 * bintime. The ffcounter is converted using the current ffclock period estimate 822 * or the "interpolated period" to ensure monotonicity. 823 * NOTE: this conversion may have been deferred, and the clock updated since the 824 * hardware counter has been read. 825 */ 826void 827ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags) 828{ 829 struct fftimehands *ffth; 830 struct bintime bt2; 831 ffcounter ffdelta; 832 uint8_t gen; 833 834 /* 835 * No locking but check generation has not changed. Also need to make 836 * sure ffdelta is positive, i.e. ffcount > tick_ffcount. 837 */ 838 do { 839 ffth = fftimehands; 840 gen = ffth->gen; 841 if (ffcount > ffth->tick_ffcount) 842 ffdelta = ffcount - ffth->tick_ffcount; 843 else 844 ffdelta = ffth->tick_ffcount - ffcount; 845 846 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) { 847 *bt = ffth->tick_time_lerp; 848 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2); 849 } else { 850 *bt = ffth->tick_time; 851 ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2); 852 } 853 854 if (ffcount > ffth->tick_ffcount) 855 bintime_add(bt, &bt2); 856 else 857 bintime_sub(bt, &bt2); 858 } while (gen == 0 || gen != ffth->gen); 859} 860 861/* 862 * Difference clock conversion. 863 * Low level function to Convert a time interval measured in RAW counter units 864 * into bintime. The difference clock allows measuring small intervals much more 865 * reliably than the absolute clock. 866 */ 867void 868ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt) 869{ 870 struct fftimehands *ffth; 871 uint8_t gen; 872 873 /* No locking but check generation has not changed. */ 874 do { 875 ffth = fftimehands; 876 gen = ffth->gen; 877 ffclock_convert_delta(ffdelta, ffth->cest.period, bt); 878 } while (gen == 0 || gen != ffth->gen); 879} 880 881/* 882 * Access to current ffcounter value. 883 */ 884void 885ffclock_read_counter(ffcounter *ffcount) 886{ 887 struct timehands *th; 888 struct fftimehands *ffth; 889 unsigned int gen, delta; 890 891 /* 892 * ffclock_windup() called from tc_windup(), safe to rely on 893 * th->th_generation only, for correct delta and ffcounter. 894 */ 895 do { 896 th = timehands; 897 gen = atomic_load_acq_int(&th->th_generation); 898 ffth = fftimehands; 899 delta = tc_delta(th); 900 *ffcount = ffth->tick_ffcount; 901 atomic_thread_fence_acq(); 902 } while (gen == 0 || gen != th->th_generation); 903 904 *ffcount += delta; 905} 906 907void 908binuptime(struct bintime *bt) 909{ 910 911 binuptime_fromclock(bt, sysclock_active); 912} 913 914void 915nanouptime(struct timespec *tsp) 916{ 917 918 nanouptime_fromclock(tsp, sysclock_active); 919} 920 921void 922microuptime(struct timeval *tvp) 923{ 924 925 microuptime_fromclock(tvp, sysclock_active); 926} 927 928void 929bintime(struct bintime *bt) 930{ 931 932 bintime_fromclock(bt, sysclock_active); 933} 934 935void 936nanotime(struct timespec *tsp) 937{ 938 939 nanotime_fromclock(tsp, sysclock_active); 940} 941 942void 943microtime(struct timeval *tvp) 944{ 945 946 microtime_fromclock(tvp, sysclock_active); 947} 948 949void 950getbinuptime(struct bintime *bt) 951{ 952 953 getbinuptime_fromclock(bt, sysclock_active); 954} 955 956void 957getnanouptime(struct timespec *tsp) 958{ 959 960 getnanouptime_fromclock(tsp, sysclock_active); 961} 962 963void 964getmicrouptime(struct timeval *tvp) 965{ 966 967 getmicrouptime_fromclock(tvp, sysclock_active); 968} 969 970void 971getbintime(struct bintime *bt) 972{ 973 974 getbintime_fromclock(bt, sysclock_active); 975} 976 977void 978getnanotime(struct timespec *tsp) 979{ 980 981 getnanotime_fromclock(tsp, sysclock_active); 982} 983 984void 985getmicrotime(struct timeval *tvp) 986{ 987 988 getmicrouptime_fromclock(tvp, sysclock_active); 989} 990 991#endif /* FFCLOCK */ 992 993/* 994 * This is a clone of getnanotime and used for walltimestamps. 995 * The dtrace_ prefix prevents fbt from creating probes for 996 * it so walltimestamp can be safely used in all fbt probes. 997 */ 998void 999dtrace_getnanotime(struct timespec *tsp) 1000{ 1001 struct timehands *th; 1002 u_int gen; 1003 1004 do { 1005 th = timehands; 1006 gen = atomic_load_acq_int(&th->th_generation); 1007 *tsp = th->th_nanotime; 1008 atomic_thread_fence_acq(); 1009 } while (gen == 0 || gen != th->th_generation); 1010} 1011 1012/* 1013 * System clock currently providing time to the system. Modifiable via sysctl 1014 * when the FFCLOCK option is defined. 1015 */ 1016int sysclock_active = SYSCLOCK_FBCK; 1017 1018/* Internal NTP status and error estimates. */ 1019extern int time_status; 1020extern long time_esterror; 1021 1022/* 1023 * Take a snapshot of sysclock data which can be used to compare system clocks 1024 * and generate timestamps after the fact. 1025 */ 1026void 1027sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast) 1028{ 1029 struct fbclock_info *fbi; 1030 struct timehands *th; 1031 struct bintime bt; 1032 unsigned int delta, gen; 1033#ifdef FFCLOCK 1034 ffcounter ffcount; 1035 struct fftimehands *ffth; 1036 struct ffclock_info *ffi; 1037 struct ffclock_estimate cest; 1038 1039 ffi = &clock_snap->ff_info; 1040#endif 1041 1042 fbi = &clock_snap->fb_info; 1043 delta = 0; 1044 1045 do { 1046 th = timehands; 1047 gen = atomic_load_acq_int(&th->th_generation); 1048 fbi->th_scale = th->th_scale; 1049 fbi->tick_time = th->th_offset; 1050#ifdef FFCLOCK 1051 ffth = fftimehands; 1052 ffi->tick_time = ffth->tick_time_lerp; 1053 ffi->tick_time_lerp = ffth->tick_time_lerp; 1054 ffi->period = ffth->cest.period; 1055 ffi->period_lerp = ffth->period_lerp; 1056 clock_snap->ffcount = ffth->tick_ffcount; 1057 cest = ffth->cest; 1058#endif 1059 if (!fast) 1060 delta = tc_delta(th); 1061 atomic_thread_fence_acq(); 1062 } while (gen == 0 || gen != th->th_generation); 1063 1064 clock_snap->delta = delta; 1065 clock_snap->sysclock_active = sysclock_active; 1066 1067 /* Record feedback clock status and error. */ 1068 clock_snap->fb_info.status = time_status; 1069 /* XXX: Very crude estimate of feedback clock error. */ 1070 bt.sec = time_esterror / 1000000; 1071 bt.frac = ((time_esterror - bt.sec) * 1000000) * 1072 (uint64_t)18446744073709ULL; 1073 clock_snap->fb_info.error = bt; 1074 1075#ifdef FFCLOCK 1076 if (!fast) 1077 clock_snap->ffcount += delta; 1078 1079 /* Record feed-forward clock leap second adjustment. */ 1080 ffi->leapsec_adjustment = cest.leapsec_total; 1081 if (clock_snap->ffcount > cest.leapsec_next) 1082 ffi->leapsec_adjustment -= cest.leapsec; 1083 1084 /* Record feed-forward clock status and error. */ 1085 clock_snap->ff_info.status = cest.status; 1086 ffcount = clock_snap->ffcount - cest.update_ffcount; 1087 ffclock_convert_delta(ffcount, cest.period, &bt); 1088 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */ 1089 bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL); 1090 /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */ 1091 bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL); 1092 clock_snap->ff_info.error = bt; 1093#endif 1094} 1095 1096/* 1097 * Convert a sysclock snapshot into a struct bintime based on the specified 1098 * clock source and flags. 1099 */ 1100int 1101sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt, 1102 int whichclock, uint32_t flags) 1103{ 1104#ifdef FFCLOCK 1105 struct bintime bt2; 1106 uint64_t period; 1107#endif 1108 1109 switch (whichclock) { 1110 case SYSCLOCK_FBCK: 1111 *bt = cs->fb_info.tick_time; 1112 1113 /* If snapshot was created with !fast, delta will be >0. */ 1114 if (cs->delta > 0) 1115 bintime_addx(bt, cs->fb_info.th_scale * cs->delta); 1116 1117 if ((flags & FBCLOCK_UPTIME) == 0) 1118 bintime_add(bt, &boottimebin); 1119 break; 1120#ifdef FFCLOCK 1121 case SYSCLOCK_FFWD: 1122 if (flags & FFCLOCK_LERP) { 1123 *bt = cs->ff_info.tick_time_lerp; 1124 period = cs->ff_info.period_lerp; 1125 } else { 1126 *bt = cs->ff_info.tick_time; 1127 period = cs->ff_info.period; 1128 } 1129 1130 /* If snapshot was created with !fast, delta will be >0. */ 1131 if (cs->delta > 0) { 1132 ffclock_convert_delta(cs->delta, period, &bt2); 1133 bintime_add(bt, &bt2); 1134 } 1135 1136 /* Leap second adjustment. */ 1137 if (flags & FFCLOCK_LEAPSEC) 1138 bt->sec -= cs->ff_info.leapsec_adjustment; 1139 1140 /* Boot time adjustment, for uptime/monotonic clocks. */ 1141 if (flags & FFCLOCK_UPTIME) 1142 bintime_sub(bt, &ffclock_boottime); 1143 break; 1144#endif 1145 default: 1146 return (EINVAL); 1147 break; 1148 } 1149 1150 return (0); 1151} 1152 1153/* 1154 * Initialize a new timecounter and possibly use it. 1155 */ 1156void 1157tc_init(struct timecounter *tc) 1158{ 1159 u_int u; 1160 struct sysctl_oid *tc_root; 1161 1162 u = tc->tc_frequency / tc->tc_counter_mask; 1163 /* XXX: We need some margin here, 10% is a guess */ 1164 u *= 11; 1165 u /= 10; 1166 if (u > hz && tc->tc_quality >= 0) { 1167 tc->tc_quality = -2000; 1168 if (bootverbose) { 1169 printf("Timecounter \"%s\" frequency %ju Hz", 1170 tc->tc_name, (uintmax_t)tc->tc_frequency); 1171 printf(" -- Insufficient hz, needs at least %u\n", u); 1172 } 1173 } else if (tc->tc_quality >= 0 || bootverbose) { 1174 printf("Timecounter \"%s\" frequency %ju Hz quality %d\n", 1175 tc->tc_name, (uintmax_t)tc->tc_frequency, 1176 tc->tc_quality); 1177 } 1178 1179 tc->tc_next = timecounters; 1180 timecounters = tc; 1181 /* 1182 * Set up sysctl tree for this counter. 1183 */ 1184 tc_root = SYSCTL_ADD_NODE(NULL, 1185 SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name, 1186 CTLFLAG_RW, 0, "timecounter description"); 1187 SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1188 "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0, 1189 "mask for implemented bits"); 1190 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1191 "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc), 1192 sysctl_kern_timecounter_get, "IU", "current timecounter value"); 1193 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1194 "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc), 1195 sysctl_kern_timecounter_freq, "QU", "timecounter frequency"); 1196 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1197 "quality", CTLFLAG_RD, &(tc->tc_quality), 0, 1198 "goodness of time counter"); 1199 /* 1200 * Never automatically use a timecounter with negative quality. 1201 * Even though we run on the dummy counter, switching here may be 1202 * worse since this timecounter may not be monotonous. 1203 */ 1204 if (tc->tc_quality < 0) 1205 return; 1206 if (tc->tc_quality < timecounter->tc_quality) 1207 return; 1208 if (tc->tc_quality == timecounter->tc_quality && 1209 tc->tc_frequency < timecounter->tc_frequency) 1210 return; 1211 (void)tc->tc_get_timecount(tc); 1212 (void)tc->tc_get_timecount(tc); 1213 timecounter = tc; 1214} 1215 1216/* Report the frequency of the current timecounter. */ 1217uint64_t 1218tc_getfrequency(void) 1219{ 1220 1221 return (timehands->th_counter->tc_frequency); 1222} 1223 1224/* 1225 * Step our concept of UTC. This is done by modifying our estimate of 1226 * when we booted. 1227 * XXX: not locked. 1228 */ 1229void 1230tc_setclock(struct timespec *ts) 1231{ 1232 struct timespec tbef, taft; 1233 struct bintime bt, bt2; 1234 1235 cpu_tick_calibrate(1); 1236 nanotime(&tbef); 1237 timespec2bintime(ts, &bt); 1238 binuptime(&bt2); 1239 bintime_sub(&bt, &bt2); 1240 bintime_add(&bt2, &boottimebin); 1241 boottimebin = bt; 1242 bintime2timeval(&bt, &boottime); 1243 1244 /* XXX fiddle all the little crinkly bits around the fiords... */ 1245 tc_windup(); 1246 nanotime(&taft); 1247 if (timestepwarnings) { 1248 log(LOG_INFO, 1249 "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n", 1250 (intmax_t)tbef.tv_sec, tbef.tv_nsec, 1251 (intmax_t)taft.tv_sec, taft.tv_nsec, 1252 (intmax_t)ts->tv_sec, ts->tv_nsec); 1253 } 1254 cpu_tick_calibrate(1); 1255} 1256 1257/* 1258 * Initialize the next struct timehands in the ring and make 1259 * it the active timehands. Along the way we might switch to a different 1260 * timecounter and/or do seconds processing in NTP. Slightly magic. 1261 */ 1262static void 1263tc_windup(void) 1264{ 1265 struct bintime bt; 1266 struct timehands *th, *tho; 1267 uint64_t scale; 1268 u_int delta, ncount, ogen; 1269 int i; 1270 time_t t; 1271 1272 /* 1273 * Make the next timehands a copy of the current one, but do 1274 * not overwrite the generation or next pointer. While we 1275 * update the contents, the generation must be zero. We need 1276 * to ensure that the zero generation is visible before the 1277 * data updates become visible, which requires release fence. 1278 * For similar reasons, re-reading of the generation after the 1279 * data is read should use acquire fence. 1280 */ 1281 tho = timehands; 1282 th = tho->th_next; 1283 ogen = th->th_generation; 1284 th->th_generation = 0; 1285 atomic_thread_fence_rel(); 1286 bcopy(tho, th, offsetof(struct timehands, th_generation)); 1287 1288 /* 1289 * Capture a timecounter delta on the current timecounter and if 1290 * changing timecounters, a counter value from the new timecounter. 1291 * Update the offset fields accordingly. 1292 */ 1293 delta = tc_delta(th); 1294 if (th->th_counter != timecounter) 1295 ncount = timecounter->tc_get_timecount(timecounter); 1296 else 1297 ncount = 0; 1298#ifdef FFCLOCK 1299 ffclock_windup(delta); 1300#endif 1301 th->th_offset_count += delta; 1302 th->th_offset_count &= th->th_counter->tc_counter_mask; 1303 while (delta > th->th_counter->tc_frequency) { 1304 /* Eat complete unadjusted seconds. */ 1305 delta -= th->th_counter->tc_frequency; 1306 th->th_offset.sec++; 1307 } 1308 if ((delta > th->th_counter->tc_frequency / 2) && 1309 (th->th_scale * delta < ((uint64_t)1 << 63))) { 1310 /* The product th_scale * delta just barely overflows. */ 1311 th->th_offset.sec++; 1312 } 1313 bintime_addx(&th->th_offset, th->th_scale * delta); 1314 1315 /* 1316 * Hardware latching timecounters may not generate interrupts on 1317 * PPS events, so instead we poll them. There is a finite risk that 1318 * the hardware might capture a count which is later than the one we 1319 * got above, and therefore possibly in the next NTP second which might 1320 * have a different rate than the current NTP second. It doesn't 1321 * matter in practice. 1322 */ 1323 if (tho->th_counter->tc_poll_pps) 1324 tho->th_counter->tc_poll_pps(tho->th_counter); 1325 1326 /* 1327 * Deal with NTP second processing. The for loop normally 1328 * iterates at most once, but in extreme situations it might 1329 * keep NTP sane if timeouts are not run for several seconds. 1330 * At boot, the time step can be large when the TOD hardware 1331 * has been read, so on really large steps, we call 1332 * ntp_update_second only twice. We need to call it twice in 1333 * case we missed a leap second. 1334 */ 1335 bt = th->th_offset; 1336 bintime_add(&bt, &boottimebin); 1337 i = bt.sec - tho->th_microtime.tv_sec; 1338 if (i > LARGE_STEP) 1339 i = 2; 1340 for (; i > 0; i--) { 1341 t = bt.sec; 1342 ntp_update_second(&th->th_adjustment, &bt.sec); 1343 if (bt.sec != t) 1344 boottimebin.sec += bt.sec - t; 1345 } 1346 /* Update the UTC timestamps used by the get*() functions. */ 1347 /* XXX shouldn't do this here. Should force non-`get' versions. */ 1348 bintime2timeval(&bt, &th->th_microtime); 1349 bintime2timespec(&bt, &th->th_nanotime); 1350 1351 /* Now is a good time to change timecounters. */ 1352 if (th->th_counter != timecounter) { 1353#ifndef __arm__ 1354 if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0) 1355 cpu_disable_c2_sleep++; 1356 if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0) 1357 cpu_disable_c2_sleep--; 1358#endif 1359 th->th_counter = timecounter; 1360 th->th_offset_count = ncount; 1361 tc_min_ticktock_freq = max(1, timecounter->tc_frequency / 1362 (((uint64_t)timecounter->tc_counter_mask + 1) / 3)); 1363#ifdef FFCLOCK 1364 ffclock_change_tc(th); 1365#endif 1366 } 1367 1368 /*- 1369 * Recalculate the scaling factor. We want the number of 1/2^64 1370 * fractions of a second per period of the hardware counter, taking 1371 * into account the th_adjustment factor which the NTP PLL/adjtime(2) 1372 * processing provides us with. 1373 * 1374 * The th_adjustment is nanoseconds per second with 32 bit binary 1375 * fraction and we want 64 bit binary fraction of second: 1376 * 1377 * x = a * 2^32 / 10^9 = a * 4.294967296 1378 * 1379 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int 1380 * we can only multiply by about 850 without overflowing, that 1381 * leaves no suitably precise fractions for multiply before divide. 1382 * 1383 * Divide before multiply with a fraction of 2199/512 results in a 1384 * systematic undercompensation of 10PPM of th_adjustment. On a 1385 * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. 1386 * 1387 * We happily sacrifice the lowest of the 64 bits of our result 1388 * to the goddess of code clarity. 1389 * 1390 */ 1391 scale = (uint64_t)1 << 63; 1392 scale += (th->th_adjustment / 1024) * 2199; 1393 scale /= th->th_counter->tc_frequency; 1394 th->th_scale = scale * 2; 1395 1396 /* 1397 * Now that the struct timehands is again consistent, set the new 1398 * generation number, making sure to not make it zero. 1399 */ 1400 if (++ogen == 0) 1401 ogen = 1; 1402 atomic_store_rel_int(&th->th_generation, ogen); 1403 1404 /* Go live with the new struct timehands. */ 1405#ifdef FFCLOCK 1406 switch (sysclock_active) { 1407 case SYSCLOCK_FBCK: 1408#endif 1409 time_second = th->th_microtime.tv_sec; 1410 time_uptime = th->th_offset.sec; 1411#ifdef FFCLOCK 1412 break; 1413 case SYSCLOCK_FFWD: 1414 time_second = fftimehands->tick_time_lerp.sec; 1415 time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec; 1416 break; 1417 } 1418#endif 1419 1420 timehands = th; 1421 timekeep_push_vdso(); 1422} 1423 1424/* Report or change the active timecounter hardware. */ 1425static int 1426sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) 1427{ 1428 char newname[32]; 1429 struct timecounter *newtc, *tc; 1430 int error; 1431 1432 tc = timecounter; 1433 strlcpy(newname, tc->tc_name, sizeof(newname)); 1434 1435 error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req); 1436 if (error != 0 || req->newptr == NULL || 1437 strcmp(newname, tc->tc_name) == 0) 1438 return (error); 1439 for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) { 1440 if (strcmp(newname, newtc->tc_name) != 0) 1441 continue; 1442 1443 /* Warm up new timecounter. */ 1444 (void)newtc->tc_get_timecount(newtc); 1445 (void)newtc->tc_get_timecount(newtc); 1446 1447 timecounter = newtc; 1448 1449 /* 1450 * The vdso timehands update is deferred until the next 1451 * 'tc_windup()'. 1452 * 1453 * This is prudent given that 'timekeep_push_vdso()' does not 1454 * use any locking and that it can be called in hard interrupt 1455 * context via 'tc_windup()'. 1456 */ 1457 return (0); 1458 } 1459 return (EINVAL); 1460} 1461 1462SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW, 1463 0, 0, sysctl_kern_timecounter_hardware, "A", 1464 "Timecounter hardware selected"); 1465 1466 1467/* Report or change the active timecounter hardware. */ 1468static int 1469sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS) 1470{ 1471 struct sbuf sb; 1472 struct timecounter *tc; 1473 int error; 1474 1475 sbuf_new_for_sysctl(&sb, NULL, 0, req); 1476 for (tc = timecounters; tc != NULL; tc = tc->tc_next) { 1477 if (tc != timecounters) 1478 sbuf_putc(&sb, ' '); 1479 sbuf_printf(&sb, "%s(%d)", tc->tc_name, tc->tc_quality); 1480 } 1481 error = sbuf_finish(&sb); 1482 sbuf_delete(&sb); 1483 return (error); 1484} 1485 1486SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD, 1487 0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected"); 1488 1489/* 1490 * RFC 2783 PPS-API implementation. 1491 */ 1492 1493/* 1494 * Return true if the driver is aware of the abi version extensions in the 1495 * pps_state structure, and it supports at least the given abi version number. 1496 */ 1497static inline int 1498abi_aware(struct pps_state *pps, int vers) 1499{ 1500 1501 return ((pps->kcmode & KCMODE_ABIFLAG) && pps->driver_abi >= vers); 1502} 1503 1504static int 1505pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps) 1506{ 1507 int err, timo; 1508 pps_seq_t aseq, cseq; 1509 struct timeval tv; 1510 1511 if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC) 1512 return (EINVAL); 1513 1514 /* 1515 * If no timeout is requested, immediately return whatever values were 1516 * most recently captured. If timeout seconds is -1, that's a request 1517 * to block without a timeout. WITNESS won't let us sleep forever 1518 * without a lock (we really don't need a lock), so just repeatedly 1519 * sleep a long time. 1520 */ 1521 if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) { 1522 if (fapi->timeout.tv_sec == -1) 1523 timo = 0x7fffffff; 1524 else { 1525 tv.tv_sec = fapi->timeout.tv_sec; 1526 tv.tv_usec = fapi->timeout.tv_nsec / 1000; 1527 timo = tvtohz(&tv); 1528 } 1529 aseq = pps->ppsinfo.assert_sequence; 1530 cseq = pps->ppsinfo.clear_sequence; 1531 while (aseq == pps->ppsinfo.assert_sequence && 1532 cseq == pps->ppsinfo.clear_sequence) { 1533 if (abi_aware(pps, 1) && pps->driver_mtx != NULL) { 1534 if (pps->flags & PPSFLAG_MTX_SPIN) { 1535 err = msleep_spin(pps, pps->driver_mtx, 1536 "ppsfch", timo); 1537 } else { 1538 err = msleep(pps, pps->driver_mtx, PCATCH, 1539 "ppsfch", timo); 1540 } 1541 } else { 1542 err = tsleep(pps, PCATCH, "ppsfch", timo); 1543 } 1544 if (err == EWOULDBLOCK) { 1545 if (fapi->timeout.tv_sec == -1) { 1546 continue; 1547 } else { 1548 return (ETIMEDOUT); 1549 } 1550 } else if (err != 0) { 1551 return (err); 1552 } 1553 } 1554 } 1555 1556 pps->ppsinfo.current_mode = pps->ppsparam.mode; 1557 fapi->pps_info_buf = pps->ppsinfo; 1558 1559 return (0); 1560} 1561 1562int 1563pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps) 1564{ 1565 pps_params_t *app; 1566 struct pps_fetch_args *fapi; 1567#ifdef FFCLOCK 1568 struct pps_fetch_ffc_args *fapi_ffc; 1569#endif 1570#ifdef PPS_SYNC 1571 struct pps_kcbind_args *kapi; 1572#endif 1573 1574 KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl")); 1575 switch (cmd) { 1576 case PPS_IOC_CREATE: 1577 return (0); 1578 case PPS_IOC_DESTROY: 1579 return (0); 1580 case PPS_IOC_SETPARAMS: 1581 app = (pps_params_t *)data; 1582 if (app->mode & ~pps->ppscap) 1583 return (EINVAL); 1584#ifdef FFCLOCK 1585 /* Ensure only a single clock is selected for ffc timestamp. */ 1586 if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK) 1587 return (EINVAL); 1588#endif 1589 pps->ppsparam = *app; 1590 return (0); 1591 case PPS_IOC_GETPARAMS: 1592 app = (pps_params_t *)data; 1593 *app = pps->ppsparam; 1594 app->api_version = PPS_API_VERS_1; 1595 return (0); 1596 case PPS_IOC_GETCAP: 1597 *(int*)data = pps->ppscap; 1598 return (0); 1599 case PPS_IOC_FETCH: 1600 fapi = (struct pps_fetch_args *)data; 1601 return (pps_fetch(fapi, pps)); 1602#ifdef FFCLOCK 1603 case PPS_IOC_FETCH_FFCOUNTER: 1604 fapi_ffc = (struct pps_fetch_ffc_args *)data; 1605 if (fapi_ffc->tsformat && fapi_ffc->tsformat != 1606 PPS_TSFMT_TSPEC) 1607 return (EINVAL); 1608 if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec) 1609 return (EOPNOTSUPP); 1610 pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode; 1611 fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc; 1612 /* Overwrite timestamps if feedback clock selected. */ 1613 switch (pps->ppsparam.mode & PPS_TSCLK_MASK) { 1614 case PPS_TSCLK_FBCK: 1615 fapi_ffc->pps_info_buf_ffc.assert_timestamp = 1616 pps->ppsinfo.assert_timestamp; 1617 fapi_ffc->pps_info_buf_ffc.clear_timestamp = 1618 pps->ppsinfo.clear_timestamp; 1619 break; 1620 case PPS_TSCLK_FFWD: 1621 break; 1622 default: 1623 break; 1624 } 1625 return (0); 1626#endif /* FFCLOCK */ 1627 case PPS_IOC_KCBIND: 1628#ifdef PPS_SYNC 1629 kapi = (struct pps_kcbind_args *)data; 1630 /* XXX Only root should be able to do this */ 1631 if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC) 1632 return (EINVAL); 1633 if (kapi->kernel_consumer != PPS_KC_HARDPPS) 1634 return (EINVAL); 1635 if (kapi->edge & ~pps->ppscap) 1636 return (EINVAL); 1637 pps->kcmode = (kapi->edge & KCMODE_EDGEMASK) | 1638 (pps->kcmode & KCMODE_ABIFLAG); 1639 return (0); 1640#else 1641 return (EOPNOTSUPP); 1642#endif 1643 default: 1644 return (ENOIOCTL); 1645 } 1646} 1647 1648void 1649pps_init(struct pps_state *pps) 1650{ 1651 pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT; 1652 if (pps->ppscap & PPS_CAPTUREASSERT) 1653 pps->ppscap |= PPS_OFFSETASSERT; 1654 if (pps->ppscap & PPS_CAPTURECLEAR) 1655 pps->ppscap |= PPS_OFFSETCLEAR; 1656#ifdef FFCLOCK 1657 pps->ppscap |= PPS_TSCLK_MASK; 1658#endif 1659 pps->kcmode &= ~KCMODE_ABIFLAG; 1660} 1661 1662void 1663pps_init_abi(struct pps_state *pps) 1664{ 1665 1666 pps_init(pps); 1667 if (pps->driver_abi > 0) { 1668 pps->kcmode |= KCMODE_ABIFLAG; 1669 pps->kernel_abi = PPS_ABI_VERSION; 1670 } 1671} 1672 1673void 1674pps_capture(struct pps_state *pps) 1675{ 1676 struct timehands *th; 1677 1678 KASSERT(pps != NULL, ("NULL pps pointer in pps_capture")); 1679 th = timehands; 1680 pps->capgen = atomic_load_acq_int(&th->th_generation); 1681 pps->capth = th; 1682#ifdef FFCLOCK 1683 pps->capffth = fftimehands; 1684#endif 1685 pps->capcount = th->th_counter->tc_get_timecount(th->th_counter); 1686 atomic_thread_fence_acq(); 1687 if (pps->capgen != th->th_generation) 1688 pps->capgen = 0; 1689} 1690 1691void 1692pps_event(struct pps_state *pps, int event) 1693{ 1694 struct bintime bt; 1695 struct timespec ts, *tsp, *osp; 1696 u_int tcount, *pcount; 1697 int foff, fhard; 1698 pps_seq_t *pseq; 1699#ifdef FFCLOCK 1700 struct timespec *tsp_ffc; 1701 pps_seq_t *pseq_ffc; 1702 ffcounter *ffcount; 1703#endif 1704 1705 KASSERT(pps != NULL, ("NULL pps pointer in pps_event")); 1706 /* If the timecounter was wound up underneath us, bail out. */ 1707 if (pps->capgen == 0 || pps->capgen != 1708 atomic_load_acq_int(&pps->capth->th_generation)) 1709 return; 1710 1711 /* Things would be easier with arrays. */ 1712 if (event == PPS_CAPTUREASSERT) { 1713 tsp = &pps->ppsinfo.assert_timestamp; 1714 osp = &pps->ppsparam.assert_offset; 1715 foff = pps->ppsparam.mode & PPS_OFFSETASSERT; 1716 fhard = pps->kcmode & PPS_CAPTUREASSERT; 1717 pcount = &pps->ppscount[0]; 1718 pseq = &pps->ppsinfo.assert_sequence; 1719#ifdef FFCLOCK 1720 ffcount = &pps->ppsinfo_ffc.assert_ffcount; 1721 tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp; 1722 pseq_ffc = &pps->ppsinfo_ffc.assert_sequence; 1723#endif 1724 } else { 1725 tsp = &pps->ppsinfo.clear_timestamp; 1726 osp = &pps->ppsparam.clear_offset; 1727 foff = pps->ppsparam.mode & PPS_OFFSETCLEAR; 1728 fhard = pps->kcmode & PPS_CAPTURECLEAR; 1729 pcount = &pps->ppscount[1]; 1730 pseq = &pps->ppsinfo.clear_sequence; 1731#ifdef FFCLOCK 1732 ffcount = &pps->ppsinfo_ffc.clear_ffcount; 1733 tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp; 1734 pseq_ffc = &pps->ppsinfo_ffc.clear_sequence; 1735#endif 1736 } 1737 1738 /* 1739 * If the timecounter changed, we cannot compare the count values, so 1740 * we have to drop the rest of the PPS-stuff until the next event. 1741 */ 1742 if (pps->ppstc != pps->capth->th_counter) { 1743 pps->ppstc = pps->capth->th_counter; 1744 *pcount = pps->capcount; 1745 pps->ppscount[2] = pps->capcount; 1746 return; 1747 } 1748 1749 /* Convert the count to a timespec. */ 1750 tcount = pps->capcount - pps->capth->th_offset_count; 1751 tcount &= pps->capth->th_counter->tc_counter_mask; 1752 bt = pps->capth->th_offset; 1753 bintime_addx(&bt, pps->capth->th_scale * tcount); 1754 bintime_add(&bt, &boottimebin); 1755 bintime2timespec(&bt, &ts); 1756 1757 /* If the timecounter was wound up underneath us, bail out. */ 1758 atomic_thread_fence_acq(); 1759 if (pps->capgen != pps->capth->th_generation) 1760 return; 1761 1762 *pcount = pps->capcount; 1763 (*pseq)++; 1764 *tsp = ts; 1765 1766 if (foff) { 1767 timespecadd(tsp, osp); 1768 if (tsp->tv_nsec < 0) { 1769 tsp->tv_nsec += 1000000000; 1770 tsp->tv_sec -= 1; 1771 } 1772 } 1773 1774#ifdef FFCLOCK 1775 *ffcount = pps->capffth->tick_ffcount + tcount; 1776 bt = pps->capffth->tick_time; 1777 ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt); 1778 bintime_add(&bt, &pps->capffth->tick_time); 1779 bintime2timespec(&bt, &ts); 1780 (*pseq_ffc)++; 1781 *tsp_ffc = ts; 1782#endif 1783 1784#ifdef PPS_SYNC 1785 if (fhard) { 1786 uint64_t scale; 1787 1788 /* 1789 * Feed the NTP PLL/FLL. 1790 * The FLL wants to know how many (hardware) nanoseconds 1791 * elapsed since the previous event. 1792 */ 1793 tcount = pps->capcount - pps->ppscount[2]; 1794 pps->ppscount[2] = pps->capcount; 1795 tcount &= pps->capth->th_counter->tc_counter_mask; 1796 scale = (uint64_t)1 << 63; 1797 scale /= pps->capth->th_counter->tc_frequency; 1798 scale *= 2; 1799 bt.sec = 0; 1800 bt.frac = 0; 1801 bintime_addx(&bt, scale * tcount); 1802 bintime2timespec(&bt, &ts); 1803 hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec); 1804 } 1805#endif 1806 1807 /* Wakeup anyone sleeping in pps_fetch(). */ 1808 wakeup(pps); 1809} 1810 1811/* 1812 * Timecounters need to be updated every so often to prevent the hardware 1813 * counter from overflowing. Updating also recalculates the cached values 1814 * used by the get*() family of functions, so their precision depends on 1815 * the update frequency. 1816 */ 1817 1818static int tc_tick; 1819SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0, 1820 "Approximate number of hardclock ticks in a millisecond"); 1821 1822void 1823tc_ticktock(int cnt) 1824{ 1825 static int count; 1826 1827 count += cnt; 1828 if (count < tc_tick) 1829 return; 1830 count = 0; 1831 tc_windup(); 1832} 1833 1834static void __inline 1835tc_adjprecision(void) 1836{ 1837 int t; 1838 1839 if (tc_timepercentage > 0) { 1840 t = (99 + tc_timepercentage) / tc_timepercentage; 1841 tc_precexp = fls(t + (t >> 1)) - 1; 1842 FREQ2BT(hz / tc_tick, &bt_timethreshold); 1843 FREQ2BT(hz, &bt_tickthreshold); 1844 bintime_shift(&bt_timethreshold, tc_precexp); 1845 bintime_shift(&bt_tickthreshold, tc_precexp); 1846 } else { 1847 tc_precexp = 31; 1848 bt_timethreshold.sec = INT_MAX; 1849 bt_timethreshold.frac = ~(uint64_t)0; 1850 bt_tickthreshold = bt_timethreshold; 1851 } 1852 sbt_timethreshold = bttosbt(bt_timethreshold); 1853 sbt_tickthreshold = bttosbt(bt_tickthreshold); 1854} 1855 1856static int 1857sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS) 1858{ 1859 int error, val; 1860 1861 val = tc_timepercentage; 1862 error = sysctl_handle_int(oidp, &val, 0, req); 1863 if (error != 0 || req->newptr == NULL) 1864 return (error); 1865 tc_timepercentage = val; 1866 if (cold) 1867 goto done; 1868 tc_adjprecision(); 1869done: 1870 return (0); 1871} 1872 1873static void 1874inittimecounter(void *dummy) 1875{ 1876 u_int p; 1877 int tick_rate; 1878 1879 /* 1880 * Set the initial timeout to 1881 * max(1, <approx. number of hardclock ticks in a millisecond>). 1882 * People should probably not use the sysctl to set the timeout 1883 * to smaller than its inital value, since that value is the 1884 * smallest reasonable one. If they want better timestamps they 1885 * should use the non-"get"* functions. 1886 */ 1887 if (hz > 1000) 1888 tc_tick = (hz + 500) / 1000; 1889 else 1890 tc_tick = 1; 1891 tc_adjprecision(); 1892 FREQ2BT(hz, &tick_bt); 1893 tick_sbt = bttosbt(tick_bt); 1894 tick_rate = hz / tc_tick; 1895 FREQ2BT(tick_rate, &tc_tick_bt); 1896 tc_tick_sbt = bttosbt(tc_tick_bt); 1897 p = (tc_tick * 1000000) / hz; 1898 printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000); 1899 1900#ifdef FFCLOCK 1901 ffclock_init(); 1902#endif 1903 /* warm up new timecounter (again) and get rolling. */ 1904 (void)timecounter->tc_get_timecount(timecounter); 1905 (void)timecounter->tc_get_timecount(timecounter); 1906 tc_windup(); 1907} 1908 1909SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL); 1910 1911/* Cpu tick handling -------------------------------------------------*/ 1912 1913static int cpu_tick_variable; 1914static uint64_t cpu_tick_frequency; 1915 1916static uint64_t 1917tc_cpu_ticks(void) 1918{ 1919 static uint64_t base; 1920 static unsigned last; 1921 unsigned u; 1922 struct timecounter *tc; 1923 1924 tc = timehands->th_counter; 1925 u = tc->tc_get_timecount(tc) & tc->tc_counter_mask; 1926 if (u < last) 1927 base += (uint64_t)tc->tc_counter_mask + 1; 1928 last = u; 1929 return (u + base); 1930} 1931 1932void 1933cpu_tick_calibration(void) 1934{ 1935 static time_t last_calib; 1936 1937 if (time_uptime != last_calib && !(time_uptime & 0xf)) { 1938 cpu_tick_calibrate(0); 1939 last_calib = time_uptime; 1940 } 1941} 1942 1943/* 1944 * This function gets called every 16 seconds on only one designated 1945 * CPU in the system from hardclock() via cpu_tick_calibration()(). 1946 * 1947 * Whenever the real time clock is stepped we get called with reset=1 1948 * to make sure we handle suspend/resume and similar events correctly. 1949 */ 1950 1951static void 1952cpu_tick_calibrate(int reset) 1953{ 1954 static uint64_t c_last; 1955 uint64_t c_this, c_delta; 1956 static struct bintime t_last; 1957 struct bintime t_this, t_delta; 1958 uint32_t divi; 1959 1960 if (reset) { 1961 /* The clock was stepped, abort & reset */ 1962 t_last.sec = 0; 1963 return; 1964 } 1965 1966 /* we don't calibrate fixed rate cputicks */ 1967 if (!cpu_tick_variable) 1968 return; 1969 1970 getbinuptime(&t_this); 1971 c_this = cpu_ticks(); 1972 if (t_last.sec != 0) { 1973 c_delta = c_this - c_last; 1974 t_delta = t_this; 1975 bintime_sub(&t_delta, &t_last); 1976 /* 1977 * Headroom: 1978 * 2^(64-20) / 16[s] = 1979 * 2^(44) / 16[s] = 1980 * 17.592.186.044.416 / 16 = 1981 * 1.099.511.627.776 [Hz] 1982 */ 1983 divi = t_delta.sec << 20; 1984 divi |= t_delta.frac >> (64 - 20); 1985 c_delta <<= 20; 1986 c_delta /= divi; 1987 if (c_delta > cpu_tick_frequency) { 1988 if (0 && bootverbose) 1989 printf("cpu_tick increased to %ju Hz\n", 1990 c_delta); 1991 cpu_tick_frequency = c_delta; 1992 } 1993 } 1994 c_last = c_this; 1995 t_last = t_this; 1996} 1997 1998void 1999set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var) 2000{ 2001 2002 if (func == NULL) { 2003 cpu_ticks = tc_cpu_ticks; 2004 } else { 2005 cpu_tick_frequency = freq; 2006 cpu_tick_variable = var; 2007 cpu_ticks = func; 2008 } 2009} 2010 2011uint64_t 2012cpu_tickrate(void) 2013{ 2014 2015 if (cpu_ticks == tc_cpu_ticks) 2016 return (tc_getfrequency()); 2017 return (cpu_tick_frequency); 2018} 2019 2020/* 2021 * We need to be slightly careful converting cputicks to microseconds. 2022 * There is plenty of margin in 64 bits of microseconds (half a million 2023 * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply 2024 * before divide conversion (to retain precision) we find that the 2025 * margin shrinks to 1.5 hours (one millionth of 146y). 2026 * With a three prong approach we never lose significant bits, no 2027 * matter what the cputick rate and length of timeinterval is. 2028 */ 2029 2030uint64_t 2031cputick2usec(uint64_t tick) 2032{ 2033 2034 if (tick > 18446744073709551LL) /* floor(2^64 / 1000) */ 2035 return (tick / (cpu_tickrate() / 1000000LL)); 2036 else if (tick > 18446744073709LL) /* floor(2^64 / 1000000) */ 2037 return ((tick * 1000LL) / (cpu_tickrate() / 1000LL)); 2038 else 2039 return ((tick * 1000000LL) / cpu_tickrate()); 2040} 2041 2042cpu_tick_f *cpu_ticks = tc_cpu_ticks; 2043 2044static int vdso_th_enable = 1; 2045static int 2046sysctl_fast_gettime(SYSCTL_HANDLER_ARGS) 2047{ 2048 int old_vdso_th_enable, error; 2049 2050 old_vdso_th_enable = vdso_th_enable; 2051 error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req); 2052 if (error != 0) 2053 return (error); 2054 vdso_th_enable = old_vdso_th_enable; 2055 return (0); 2056} 2057SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime, 2058 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 2059 NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day"); 2060 2061uint32_t 2062tc_fill_vdso_timehands(struct vdso_timehands *vdso_th) 2063{ 2064 struct timehands *th; 2065 uint32_t enabled; 2066 2067 th = timehands; 2068 vdso_th->th_algo = VDSO_TH_ALGO_1; 2069 vdso_th->th_scale = th->th_scale; 2070 vdso_th->th_offset_count = th->th_offset_count; 2071 vdso_th->th_counter_mask = th->th_counter->tc_counter_mask; 2072 vdso_th->th_offset = th->th_offset; 2073 vdso_th->th_boottime = boottimebin; 2074 enabled = cpu_fill_vdso_timehands(vdso_th, th->th_counter); 2075 if (!vdso_th_enable) 2076 enabled = 0; 2077 return (enabled); 2078} 2079 2080#ifdef COMPAT_FREEBSD32 2081uint32_t 2082tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) 2083{ 2084 struct timehands *th; 2085 uint32_t enabled; 2086 2087 th = timehands; 2088 vdso_th32->th_algo = VDSO_TH_ALGO_1; 2089 *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale; 2090 vdso_th32->th_offset_count = th->th_offset_count; 2091 vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask; 2092 vdso_th32->th_offset.sec = th->th_offset.sec; 2093 *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac; 2094 vdso_th32->th_boottime.sec = boottimebin.sec; 2095 *(uint64_t *)&vdso_th32->th_boottime.frac[0] = boottimebin.frac; 2096 enabled = cpu_fill_vdso_timehands32(vdso_th32, th->th_counter); 2097 if (!vdso_th_enable) 2098 enabled = 0; 2099 return (enabled); 2100} 2101#endif 2102