1/*- 2 * ---------------------------------------------------------------------------- 3 * "THE BEER-WARE LICENSE" (Revision 42): 4 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you 5 * can do whatever you want with this stuff. If we meet some day, and you think 6 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 7 * ---------------------------------------------------------------------------- 8 * 9 * Copyright (c) 2011 The FreeBSD Foundation 10 * All rights reserved. 11 * 12 * Portions of this software were developed by Julien Ridoux at the University 13 * of Melbourne under sponsorship from the FreeBSD Foundation. 14 */ 15 16#include <sys/cdefs.h> 17__FBSDID("$FreeBSD$"); 18 19#include "opt_compat.h" 20#include "opt_ntp.h" 21#include "opt_ffclock.h" 22 23#include <sys/param.h> 24#include <sys/kernel.h> 25#include <sys/limits.h> 26#include <sys/lock.h> 27#include <sys/mutex.h> 28#include <sys/sysctl.h> 29#include <sys/syslog.h> 30#include <sys/systm.h> 31#include <sys/timeffc.h> 32#include <sys/timepps.h> 33#include <sys/timetc.h> 34#include <sys/timex.h> 35#include <sys/vdso.h> 36 37/* 38 * A large step happens on boot. This constant detects such steps. 39 * It is relatively small so that ntp_update_second gets called enough 40 * in the typical 'missed a couple of seconds' case, but doesn't loop 41 * forever when the time step is large. 42 */ 43#define LARGE_STEP 200 44 45/* 46 * Implement a dummy timecounter which we can use until we get a real one 47 * in the air. This allows the console and other early stuff to use 48 * time services. 49 */ 50 51static u_int 52dummy_get_timecount(struct timecounter *tc) 53{ 54 static u_int now; 55 56 return (++now); 57} 58 59static struct timecounter dummy_timecounter = { 60 dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000 61}; 62 63struct timehands { 64 /* These fields must be initialized by the driver. */ 65 struct timecounter *th_counter; 66 int64_t th_adjustment; 67 uint64_t th_scale; 68 u_int th_offset_count; 69 struct bintime th_offset; 70 struct timeval th_microtime; 71 struct timespec th_nanotime; 72 /* Fields not to be copied in tc_windup start with th_generation. */ 73 volatile u_int th_generation; 74 struct timehands *th_next; 75}; 76 77static struct timehands th0; 78static struct timehands th9 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th0}; 79static struct timehands th8 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th9}; 80static struct timehands th7 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th8}; 81static struct timehands th6 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th7}; 82static struct timehands th5 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th6}; 83static struct timehands th4 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th5}; 84static struct timehands th3 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th4}; 85static struct timehands th2 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th3}; 86static struct timehands th1 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th2}; 87static struct timehands th0 = { 88 &dummy_timecounter, 89 0, 90 (uint64_t)-1 / 1000000, 91 0, 92 {1, 0}, 93 {0, 0}, 94 {0, 0}, 95 1, 96 &th1 97}; 98 99static struct timehands *volatile timehands = &th0; 100struct timecounter *timecounter = &dummy_timecounter; 101static struct timecounter *timecounters = &dummy_timecounter; 102 103int tc_min_ticktock_freq = 1; 104 105volatile time_t time_second = 1; 106volatile time_t time_uptime = 1; 107 108struct bintime boottimebin; 109struct timeval boottime; 110static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS); 111SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD, 112 NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime"); 113 114SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, ""); 115static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, ""); 116 117static int timestepwarnings; 118SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW, 119 ×tepwarnings, 0, "Log time steps"); 120 121struct bintime bt_timethreshold; 122struct bintime bt_tickthreshold; 123sbintime_t sbt_timethreshold; 124sbintime_t sbt_tickthreshold; 125struct bintime tc_tick_bt; 126sbintime_t tc_tick_sbt; 127int tc_precexp; 128int tc_timepercentage = TC_DEFAULTPERC; 129TUNABLE_INT("kern.timecounter.alloweddeviation", &tc_timepercentage); 130static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS); 131SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation, 132 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, 133 sysctl_kern_timecounter_adjprecision, "I", 134 "Allowed time interval deviation in percents"); 135 136static int tc_chosen; /* Non-zero if a specific tc was chosen via sysctl. */ 137 138static void tc_windup(void); 139static void cpu_tick_calibrate(int); 140 141void dtrace_getnanotime(struct timespec *tsp); 142 143static int 144sysctl_kern_boottime(SYSCTL_HANDLER_ARGS) 145{ 146#ifndef __mips__ 147#ifdef SCTL_MASK32 148 int tv[2]; 149 150 if (req->flags & SCTL_MASK32) { 151 tv[0] = boottime.tv_sec; 152 tv[1] = boottime.tv_usec; 153 return SYSCTL_OUT(req, tv, sizeof(tv)); 154 } else 155#endif 156#endif 157 return SYSCTL_OUT(req, &boottime, sizeof(boottime)); 158} 159 160static int 161sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS) 162{ 163 u_int ncount; 164 struct timecounter *tc = arg1; 165 166 ncount = tc->tc_get_timecount(tc); 167 return sysctl_handle_int(oidp, &ncount, 0, req); 168} 169 170static int 171sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS) 172{ 173 uint64_t freq; 174 struct timecounter *tc = arg1; 175 176 freq = tc->tc_frequency; 177 return sysctl_handle_64(oidp, &freq, 0, req); 178} 179 180/* 181 * Return the difference between the timehands' counter value now and what 182 * was when we copied it to the timehands' offset_count. 183 */ 184static __inline u_int 185tc_delta(struct timehands *th) 186{ 187 struct timecounter *tc; 188 189 tc = th->th_counter; 190 return ((tc->tc_get_timecount(tc) - th->th_offset_count) & 191 tc->tc_counter_mask); 192} 193 194/* 195 * Functions for reading the time. We have to loop until we are sure that 196 * the timehands that we operated on was not updated under our feet. See 197 * the comment in <sys/time.h> for a description of these 12 functions. 198 */ 199 200#ifdef FFCLOCK 201void 202fbclock_binuptime(struct bintime *bt) 203{ 204 struct timehands *th; 205 unsigned int gen; 206 207 do { 208 th = timehands; 209 gen = th->th_generation; 210 *bt = th->th_offset; 211 bintime_addx(bt, th->th_scale * tc_delta(th)); 212 } while (gen == 0 || gen != th->th_generation); 213} 214 215void 216fbclock_nanouptime(struct timespec *tsp) 217{ 218 struct bintime bt; 219 220 fbclock_binuptime(&bt); 221 bintime2timespec(&bt, tsp); 222} 223 224void 225fbclock_microuptime(struct timeval *tvp) 226{ 227 struct bintime bt; 228 229 fbclock_binuptime(&bt); 230 bintime2timeval(&bt, tvp); 231} 232 233void 234fbclock_bintime(struct bintime *bt) 235{ 236 237 fbclock_binuptime(bt); 238 bintime_add(bt, &boottimebin); 239} 240 241void 242fbclock_nanotime(struct timespec *tsp) 243{ 244 struct bintime bt; 245 246 fbclock_bintime(&bt); 247 bintime2timespec(&bt, tsp); 248} 249 250void 251fbclock_microtime(struct timeval *tvp) 252{ 253 struct bintime bt; 254 255 fbclock_bintime(&bt); 256 bintime2timeval(&bt, tvp); 257} 258 259void 260fbclock_getbinuptime(struct bintime *bt) 261{ 262 struct timehands *th; 263 unsigned int gen; 264 265 do { 266 th = timehands; 267 gen = th->th_generation; 268 *bt = th->th_offset; 269 } while (gen == 0 || gen != th->th_generation); 270} 271 272void 273fbclock_getnanouptime(struct timespec *tsp) 274{ 275 struct timehands *th; 276 unsigned int gen; 277 278 do { 279 th = timehands; 280 gen = th->th_generation; 281 bintime2timespec(&th->th_offset, tsp); 282 } while (gen == 0 || gen != th->th_generation); 283} 284 285void 286fbclock_getmicrouptime(struct timeval *tvp) 287{ 288 struct timehands *th; 289 unsigned int gen; 290 291 do { 292 th = timehands; 293 gen = th->th_generation; 294 bintime2timeval(&th->th_offset, tvp); 295 } while (gen == 0 || gen != th->th_generation); 296} 297 298void 299fbclock_getbintime(struct bintime *bt) 300{ 301 struct timehands *th; 302 unsigned int gen; 303 304 do { 305 th = timehands; 306 gen = th->th_generation; 307 *bt = th->th_offset; 308 } while (gen == 0 || gen != th->th_generation); 309 bintime_add(bt, &boottimebin); 310} 311 312void 313fbclock_getnanotime(struct timespec *tsp) 314{ 315 struct timehands *th; 316 unsigned int gen; 317 318 do { 319 th = timehands; 320 gen = th->th_generation; 321 *tsp = th->th_nanotime; 322 } while (gen == 0 || gen != th->th_generation); 323} 324 325void 326fbclock_getmicrotime(struct timeval *tvp) 327{ 328 struct timehands *th; 329 unsigned int gen; 330 331 do { 332 th = timehands; 333 gen = th->th_generation; 334 *tvp = th->th_microtime; 335 } while (gen == 0 || gen != th->th_generation); 336} 337#else /* !FFCLOCK */ 338void 339binuptime(struct bintime *bt) 340{ 341 struct timehands *th; 342 u_int gen; 343 344 do { 345 th = timehands; 346 gen = th->th_generation; 347 *bt = th->th_offset; 348 bintime_addx(bt, th->th_scale * tc_delta(th)); 349 } while (gen == 0 || gen != th->th_generation); 350} 351 352void 353nanouptime(struct timespec *tsp) 354{ 355 struct bintime bt; 356 357 binuptime(&bt); 358 bintime2timespec(&bt, tsp); 359} 360 361void 362microuptime(struct timeval *tvp) 363{ 364 struct bintime bt; 365 366 binuptime(&bt); 367 bintime2timeval(&bt, tvp); 368} 369 370void 371bintime(struct bintime *bt) 372{ 373 374 binuptime(bt); 375 bintime_add(bt, &boottimebin); 376} 377 378void 379nanotime(struct timespec *tsp) 380{ 381 struct bintime bt; 382 383 bintime(&bt); 384 bintime2timespec(&bt, tsp); 385} 386 387void 388microtime(struct timeval *tvp) 389{ 390 struct bintime bt; 391 392 bintime(&bt); 393 bintime2timeval(&bt, tvp); 394} 395 396void 397getbinuptime(struct bintime *bt) 398{ 399 struct timehands *th; 400 u_int gen; 401 402 do { 403 th = timehands; 404 gen = th->th_generation; 405 *bt = th->th_offset; 406 } while (gen == 0 || gen != th->th_generation); 407} 408 409void 410getnanouptime(struct timespec *tsp) 411{ 412 struct timehands *th; 413 u_int gen; 414 415 do { 416 th = timehands; 417 gen = th->th_generation; 418 bintime2timespec(&th->th_offset, tsp); 419 } while (gen == 0 || gen != th->th_generation); 420} 421 422void 423getmicrouptime(struct timeval *tvp) 424{ 425 struct timehands *th; 426 u_int gen; 427 428 do { 429 th = timehands; 430 gen = th->th_generation; 431 bintime2timeval(&th->th_offset, tvp); 432 } while (gen == 0 || gen != th->th_generation); 433} 434 435void 436getbintime(struct bintime *bt) 437{ 438 struct timehands *th; 439 u_int gen; 440 441 do { 442 th = timehands; 443 gen = th->th_generation; 444 *bt = th->th_offset; 445 } while (gen == 0 || gen != th->th_generation); 446 bintime_add(bt, &boottimebin); 447} 448 449void 450getnanotime(struct timespec *tsp) 451{ 452 struct timehands *th; 453 u_int gen; 454 455 do { 456 th = timehands; 457 gen = th->th_generation; 458 *tsp = th->th_nanotime; 459 } while (gen == 0 || gen != th->th_generation); 460} 461 462void 463getmicrotime(struct timeval *tvp) 464{ 465 struct timehands *th; 466 u_int gen; 467 468 do { 469 th = timehands; 470 gen = th->th_generation; 471 *tvp = th->th_microtime; 472 } while (gen == 0 || gen != th->th_generation); 473} 474#endif /* FFCLOCK */ 475 476#ifdef FFCLOCK 477/* 478 * Support for feed-forward synchronization algorithms. This is heavily inspired 479 * by the timehands mechanism but kept independent from it. *_windup() functions 480 * have some connection to avoid accessing the timecounter hardware more than 481 * necessary. 482 */ 483 484/* Feed-forward clock estimates kept updated by the synchronization daemon. */ 485struct ffclock_estimate ffclock_estimate; 486struct bintime ffclock_boottime; /* Feed-forward boot time estimate. */ 487uint32_t ffclock_status; /* Feed-forward clock status. */ 488int8_t ffclock_updated; /* New estimates are available. */ 489struct mtx ffclock_mtx; /* Mutex on ffclock_estimate. */ 490 491struct fftimehands { 492 struct ffclock_estimate cest; 493 struct bintime tick_time; 494 struct bintime tick_time_lerp; 495 ffcounter tick_ffcount; 496 uint64_t period_lerp; 497 volatile uint8_t gen; 498 struct fftimehands *next; 499}; 500 501#define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x)) 502 503static struct fftimehands ffth[10]; 504static struct fftimehands *volatile fftimehands = ffth; 505 506static void 507ffclock_init(void) 508{ 509 struct fftimehands *cur; 510 struct fftimehands *last; 511 512 memset(ffth, 0, sizeof(ffth)); 513 514 last = ffth + NUM_ELEMENTS(ffth) - 1; 515 for (cur = ffth; cur < last; cur++) 516 cur->next = cur + 1; 517 last->next = ffth; 518 519 ffclock_updated = 0; 520 ffclock_status = FFCLOCK_STA_UNSYNC; 521 mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF); 522} 523 524/* 525 * Reset the feed-forward clock estimates. Called from inittodr() to get things 526 * kick started and uses the timecounter nominal frequency as a first period 527 * estimate. Note: this function may be called several time just after boot. 528 * Note: this is the only function that sets the value of boot time for the 529 * monotonic (i.e. uptime) version of the feed-forward clock. 530 */ 531void 532ffclock_reset_clock(struct timespec *ts) 533{ 534 struct timecounter *tc; 535 struct ffclock_estimate cest; 536 537 tc = timehands->th_counter; 538 memset(&cest, 0, sizeof(struct ffclock_estimate)); 539 540 timespec2bintime(ts, &ffclock_boottime); 541 timespec2bintime(ts, &(cest.update_time)); 542 ffclock_read_counter(&cest.update_ffcount); 543 cest.leapsec_next = 0; 544 cest.period = ((1ULL << 63) / tc->tc_frequency) << 1; 545 cest.errb_abs = 0; 546 cest.errb_rate = 0; 547 cest.status = FFCLOCK_STA_UNSYNC; 548 cest.leapsec_total = 0; 549 cest.leapsec = 0; 550 551 mtx_lock(&ffclock_mtx); 552 bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate)); 553 ffclock_updated = INT8_MAX; 554 mtx_unlock(&ffclock_mtx); 555 556 printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name, 557 (unsigned long long)tc->tc_frequency, (long)ts->tv_sec, 558 (unsigned long)ts->tv_nsec); 559} 560 561/* 562 * Sub-routine to convert a time interval measured in RAW counter units to time 563 * in seconds stored in bintime format. 564 * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be 565 * larger than the max value of u_int (on 32 bit architecture). Loop to consume 566 * extra cycles. 567 */ 568static void 569ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt) 570{ 571 struct bintime bt2; 572 ffcounter delta, delta_max; 573 574 delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1; 575 bintime_clear(bt); 576 do { 577 if (ffdelta > delta_max) 578 delta = delta_max; 579 else 580 delta = ffdelta; 581 bt2.sec = 0; 582 bt2.frac = period; 583 bintime_mul(&bt2, (unsigned int)delta); 584 bintime_add(bt, &bt2); 585 ffdelta -= delta; 586 } while (ffdelta > 0); 587} 588 589/* 590 * Update the fftimehands. 591 * Push the tick ffcount and time(s) forward based on current clock estimate. 592 * The conversion from ffcounter to bintime relies on the difference clock 593 * principle, whose accuracy relies on computing small time intervals. If a new 594 * clock estimate has been passed by the synchronisation daemon, make it 595 * current, and compute the linear interpolation for monotonic time if needed. 596 */ 597static void 598ffclock_windup(unsigned int delta) 599{ 600 struct ffclock_estimate *cest; 601 struct fftimehands *ffth; 602 struct bintime bt, gap_lerp; 603 ffcounter ffdelta; 604 uint64_t frac; 605 unsigned int polling; 606 uint8_t forward_jump, ogen; 607 608 /* 609 * Pick the next timehand, copy current ffclock estimates and move tick 610 * times and counter forward. 611 */ 612 forward_jump = 0; 613 ffth = fftimehands->next; 614 ogen = ffth->gen; 615 ffth->gen = 0; 616 cest = &ffth->cest; 617 bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate)); 618 ffdelta = (ffcounter)delta; 619 ffth->period_lerp = fftimehands->period_lerp; 620 621 ffth->tick_time = fftimehands->tick_time; 622 ffclock_convert_delta(ffdelta, cest->period, &bt); 623 bintime_add(&ffth->tick_time, &bt); 624 625 ffth->tick_time_lerp = fftimehands->tick_time_lerp; 626 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt); 627 bintime_add(&ffth->tick_time_lerp, &bt); 628 629 ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta; 630 631 /* 632 * Assess the status of the clock, if the last update is too old, it is 633 * likely the synchronisation daemon is dead and the clock is free 634 * running. 635 */ 636 if (ffclock_updated == 0) { 637 ffdelta = ffth->tick_ffcount - cest->update_ffcount; 638 ffclock_convert_delta(ffdelta, cest->period, &bt); 639 if (bt.sec > 2 * FFCLOCK_SKM_SCALE) 640 ffclock_status |= FFCLOCK_STA_UNSYNC; 641 } 642 643 /* 644 * If available, grab updated clock estimates and make them current. 645 * Recompute time at this tick using the updated estimates. The clock 646 * estimates passed the feed-forward synchronisation daemon may result 647 * in time conversion that is not monotonically increasing (just after 648 * the update). time_lerp is a particular linear interpolation over the 649 * synchronisation algo polling period that ensures monotonicity for the 650 * clock ids requesting it. 651 */ 652 if (ffclock_updated > 0) { 653 bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate)); 654 ffdelta = ffth->tick_ffcount - cest->update_ffcount; 655 ffth->tick_time = cest->update_time; 656 ffclock_convert_delta(ffdelta, cest->period, &bt); 657 bintime_add(&ffth->tick_time, &bt); 658 659 /* ffclock_reset sets ffclock_updated to INT8_MAX */ 660 if (ffclock_updated == INT8_MAX) 661 ffth->tick_time_lerp = ffth->tick_time; 662 663 if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >)) 664 forward_jump = 1; 665 else 666 forward_jump = 0; 667 668 bintime_clear(&gap_lerp); 669 if (forward_jump) { 670 gap_lerp = ffth->tick_time; 671 bintime_sub(&gap_lerp, &ffth->tick_time_lerp); 672 } else { 673 gap_lerp = ffth->tick_time_lerp; 674 bintime_sub(&gap_lerp, &ffth->tick_time); 675 } 676 677 /* 678 * The reset from the RTC clock may be far from accurate, and 679 * reducing the gap between real time and interpolated time 680 * could take a very long time if the interpolated clock insists 681 * on strict monotonicity. The clock is reset under very strict 682 * conditions (kernel time is known to be wrong and 683 * synchronization daemon has been restarted recently. 684 * ffclock_boottime absorbs the jump to ensure boot time is 685 * correct and uptime functions stay consistent. 686 */ 687 if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) && 688 ((cest->status & FFCLOCK_STA_UNSYNC) == 0) && 689 ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) { 690 if (forward_jump) 691 bintime_add(&ffclock_boottime, &gap_lerp); 692 else 693 bintime_sub(&ffclock_boottime, &gap_lerp); 694 ffth->tick_time_lerp = ffth->tick_time; 695 bintime_clear(&gap_lerp); 696 } 697 698 ffclock_status = cest->status; 699 ffth->period_lerp = cest->period; 700 701 /* 702 * Compute corrected period used for the linear interpolation of 703 * time. The rate of linear interpolation is capped to 5000PPM 704 * (5ms/s). 705 */ 706 if (bintime_isset(&gap_lerp)) { 707 ffdelta = cest->update_ffcount; 708 ffdelta -= fftimehands->cest.update_ffcount; 709 ffclock_convert_delta(ffdelta, cest->period, &bt); 710 polling = bt.sec; 711 bt.sec = 0; 712 bt.frac = 5000000 * (uint64_t)18446744073LL; 713 bintime_mul(&bt, polling); 714 if (bintime_cmp(&gap_lerp, &bt, >)) 715 gap_lerp = bt; 716 717 /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */ 718 frac = 0; 719 if (gap_lerp.sec > 0) { 720 frac -= 1; 721 frac /= ffdelta / gap_lerp.sec; 722 } 723 frac += gap_lerp.frac / ffdelta; 724 725 if (forward_jump) 726 ffth->period_lerp += frac; 727 else 728 ffth->period_lerp -= frac; 729 } 730 731 ffclock_updated = 0; 732 } 733 if (++ogen == 0) 734 ogen = 1; 735 ffth->gen = ogen; 736 fftimehands = ffth; 737} 738 739/* 740 * Adjust the fftimehands when the timecounter is changed. Stating the obvious, 741 * the old and new hardware counter cannot be read simultaneously. tc_windup() 742 * does read the two counters 'back to back', but a few cycles are effectively 743 * lost, and not accumulated in tick_ffcount. This is a fairly radical 744 * operation for a feed-forward synchronization daemon, and it is its job to not 745 * pushing irrelevant data to the kernel. Because there is no locking here, 746 * simply force to ignore pending or next update to give daemon a chance to 747 * realize the counter has changed. 748 */ 749static void 750ffclock_change_tc(struct timehands *th) 751{ 752 struct fftimehands *ffth; 753 struct ffclock_estimate *cest; 754 struct timecounter *tc; 755 uint8_t ogen; 756 757 tc = th->th_counter; 758 ffth = fftimehands->next; 759 ogen = ffth->gen; 760 ffth->gen = 0; 761 762 cest = &ffth->cest; 763 bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate)); 764 cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1; 765 cest->errb_abs = 0; 766 cest->errb_rate = 0; 767 cest->status |= FFCLOCK_STA_UNSYNC; 768 769 ffth->tick_ffcount = fftimehands->tick_ffcount; 770 ffth->tick_time_lerp = fftimehands->tick_time_lerp; 771 ffth->tick_time = fftimehands->tick_time; 772 ffth->period_lerp = cest->period; 773 774 /* Do not lock but ignore next update from synchronization daemon. */ 775 ffclock_updated--; 776 777 if (++ogen == 0) 778 ogen = 1; 779 ffth->gen = ogen; 780 fftimehands = ffth; 781} 782 783/* 784 * Retrieve feed-forward counter and time of last kernel tick. 785 */ 786void 787ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags) 788{ 789 struct fftimehands *ffth; 790 uint8_t gen; 791 792 /* 793 * No locking but check generation has not changed. Also need to make 794 * sure ffdelta is positive, i.e. ffcount > tick_ffcount. 795 */ 796 do { 797 ffth = fftimehands; 798 gen = ffth->gen; 799 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) 800 *bt = ffth->tick_time_lerp; 801 else 802 *bt = ffth->tick_time; 803 *ffcount = ffth->tick_ffcount; 804 } while (gen == 0 || gen != ffth->gen); 805} 806 807/* 808 * Absolute clock conversion. Low level function to convert ffcounter to 809 * bintime. The ffcounter is converted using the current ffclock period estimate 810 * or the "interpolated period" to ensure monotonicity. 811 * NOTE: this conversion may have been deferred, and the clock updated since the 812 * hardware counter has been read. 813 */ 814void 815ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags) 816{ 817 struct fftimehands *ffth; 818 struct bintime bt2; 819 ffcounter ffdelta; 820 uint8_t gen; 821 822 /* 823 * No locking but check generation has not changed. Also need to make 824 * sure ffdelta is positive, i.e. ffcount > tick_ffcount. 825 */ 826 do { 827 ffth = fftimehands; 828 gen = ffth->gen; 829 if (ffcount > ffth->tick_ffcount) 830 ffdelta = ffcount - ffth->tick_ffcount; 831 else 832 ffdelta = ffth->tick_ffcount - ffcount; 833 834 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) { 835 *bt = ffth->tick_time_lerp; 836 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2); 837 } else { 838 *bt = ffth->tick_time; 839 ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2); 840 } 841 842 if (ffcount > ffth->tick_ffcount) 843 bintime_add(bt, &bt2); 844 else 845 bintime_sub(bt, &bt2); 846 } while (gen == 0 || gen != ffth->gen); 847} 848 849/* 850 * Difference clock conversion. 851 * Low level function to Convert a time interval measured in RAW counter units 852 * into bintime. The difference clock allows measuring small intervals much more 853 * reliably than the absolute clock. 854 */ 855void 856ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt) 857{ 858 struct fftimehands *ffth; 859 uint8_t gen; 860 861 /* No locking but check generation has not changed. */ 862 do { 863 ffth = fftimehands; 864 gen = ffth->gen; 865 ffclock_convert_delta(ffdelta, ffth->cest.period, bt); 866 } while (gen == 0 || gen != ffth->gen); 867} 868 869/* 870 * Access to current ffcounter value. 871 */ 872void 873ffclock_read_counter(ffcounter *ffcount) 874{ 875 struct timehands *th; 876 struct fftimehands *ffth; 877 unsigned int gen, delta; 878 879 /* 880 * ffclock_windup() called from tc_windup(), safe to rely on 881 * th->th_generation only, for correct delta and ffcounter. 882 */ 883 do { 884 th = timehands; 885 gen = th->th_generation; 886 ffth = fftimehands; 887 delta = tc_delta(th); 888 *ffcount = ffth->tick_ffcount; 889 } while (gen == 0 || gen != th->th_generation); 890 891 *ffcount += delta; 892} 893 894void 895binuptime(struct bintime *bt) 896{ 897 898 binuptime_fromclock(bt, sysclock_active); 899} 900 901void 902nanouptime(struct timespec *tsp) 903{ 904 905 nanouptime_fromclock(tsp, sysclock_active); 906} 907 908void 909microuptime(struct timeval *tvp) 910{ 911 912 microuptime_fromclock(tvp, sysclock_active); 913} 914 915void 916bintime(struct bintime *bt) 917{ 918 919 bintime_fromclock(bt, sysclock_active); 920} 921 922void 923nanotime(struct timespec *tsp) 924{ 925 926 nanotime_fromclock(tsp, sysclock_active); 927} 928 929void 930microtime(struct timeval *tvp) 931{ 932 933 microtime_fromclock(tvp, sysclock_active); 934} 935 936void 937getbinuptime(struct bintime *bt) 938{ 939 940 getbinuptime_fromclock(bt, sysclock_active); 941} 942 943void 944getnanouptime(struct timespec *tsp) 945{ 946 947 getnanouptime_fromclock(tsp, sysclock_active); 948} 949 950void 951getmicrouptime(struct timeval *tvp) 952{ 953 954 getmicrouptime_fromclock(tvp, sysclock_active); 955} 956 957void 958getbintime(struct bintime *bt) 959{ 960 961 getbintime_fromclock(bt, sysclock_active); 962} 963 964void 965getnanotime(struct timespec *tsp) 966{ 967 968 getnanotime_fromclock(tsp, sysclock_active); 969} 970 971void 972getmicrotime(struct timeval *tvp) 973{ 974 975 getmicrouptime_fromclock(tvp, sysclock_active); 976} 977 978#endif /* FFCLOCK */ 979 980/* 981 * This is a clone of getnanotime and used for walltimestamps. 982 * The dtrace_ prefix prevents fbt from creating probes for 983 * it so walltimestamp can be safely used in all fbt probes. 984 */ 985void 986dtrace_getnanotime(struct timespec *tsp) 987{ 988 struct timehands *th; 989 u_int gen; 990 991 do { 992 th = timehands; 993 gen = th->th_generation; 994 *tsp = th->th_nanotime; 995 } while (gen == 0 || gen != th->th_generation); 996} 997 998/* 999 * System clock currently providing time to the system. Modifiable via sysctl 1000 * when the FFCLOCK option is defined. 1001 */ 1002int sysclock_active = SYSCLOCK_FBCK; 1003 1004/* Internal NTP status and error estimates. */ 1005extern int time_status; 1006extern long time_esterror; 1007 1008/* 1009 * Take a snapshot of sysclock data which can be used to compare system clocks 1010 * and generate timestamps after the fact. 1011 */ 1012void 1013sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast) 1014{ 1015 struct fbclock_info *fbi; 1016 struct timehands *th; 1017 struct bintime bt; 1018 unsigned int delta, gen; 1019#ifdef FFCLOCK 1020 ffcounter ffcount; 1021 struct fftimehands *ffth; 1022 struct ffclock_info *ffi; 1023 struct ffclock_estimate cest; 1024 1025 ffi = &clock_snap->ff_info; 1026#endif 1027 1028 fbi = &clock_snap->fb_info; 1029 delta = 0; 1030 1031 do { 1032 th = timehands; 1033 gen = th->th_generation; 1034 fbi->th_scale = th->th_scale; 1035 fbi->tick_time = th->th_offset; 1036#ifdef FFCLOCK 1037 ffth = fftimehands; 1038 ffi->tick_time = ffth->tick_time_lerp; 1039 ffi->tick_time_lerp = ffth->tick_time_lerp; 1040 ffi->period = ffth->cest.period; 1041 ffi->period_lerp = ffth->period_lerp; 1042 clock_snap->ffcount = ffth->tick_ffcount; 1043 cest = ffth->cest; 1044#endif 1045 if (!fast) 1046 delta = tc_delta(th); 1047 } while (gen == 0 || gen != th->th_generation); 1048 1049 clock_snap->delta = delta; 1050 clock_snap->sysclock_active = sysclock_active; 1051 1052 /* Record feedback clock status and error. */ 1053 clock_snap->fb_info.status = time_status; 1054 /* XXX: Very crude estimate of feedback clock error. */ 1055 bt.sec = time_esterror / 1000000; 1056 bt.frac = ((time_esterror - bt.sec) * 1000000) * 1057 (uint64_t)18446744073709ULL; 1058 clock_snap->fb_info.error = bt; 1059 1060#ifdef FFCLOCK 1061 if (!fast) 1062 clock_snap->ffcount += delta; 1063 1064 /* Record feed-forward clock leap second adjustment. */ 1065 ffi->leapsec_adjustment = cest.leapsec_total; 1066 if (clock_snap->ffcount > cest.leapsec_next) 1067 ffi->leapsec_adjustment -= cest.leapsec; 1068 1069 /* Record feed-forward clock status and error. */ 1070 clock_snap->ff_info.status = cest.status; 1071 ffcount = clock_snap->ffcount - cest.update_ffcount; 1072 ffclock_convert_delta(ffcount, cest.period, &bt); 1073 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */ 1074 bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL); 1075 /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */ 1076 bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL); 1077 clock_snap->ff_info.error = bt; 1078#endif 1079} 1080 1081/* 1082 * Convert a sysclock snapshot into a struct bintime based on the specified 1083 * clock source and flags. 1084 */ 1085int 1086sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt, 1087 int whichclock, uint32_t flags) 1088{ 1089#ifdef FFCLOCK 1090 struct bintime bt2; 1091 uint64_t period; 1092#endif 1093 1094 switch (whichclock) { 1095 case SYSCLOCK_FBCK: 1096 *bt = cs->fb_info.tick_time; 1097 1098 /* If snapshot was created with !fast, delta will be >0. */ 1099 if (cs->delta > 0) 1100 bintime_addx(bt, cs->fb_info.th_scale * cs->delta); 1101 1102 if ((flags & FBCLOCK_UPTIME) == 0) 1103 bintime_add(bt, &boottimebin); 1104 break; 1105#ifdef FFCLOCK 1106 case SYSCLOCK_FFWD: 1107 if (flags & FFCLOCK_LERP) { 1108 *bt = cs->ff_info.tick_time_lerp; 1109 period = cs->ff_info.period_lerp; 1110 } else { 1111 *bt = cs->ff_info.tick_time; 1112 period = cs->ff_info.period; 1113 } 1114 1115 /* If snapshot was created with !fast, delta will be >0. */ 1116 if (cs->delta > 0) { 1117 ffclock_convert_delta(cs->delta, period, &bt2); 1118 bintime_add(bt, &bt2); 1119 } 1120 1121 /* Leap second adjustment. */ 1122 if (flags & FFCLOCK_LEAPSEC) 1123 bt->sec -= cs->ff_info.leapsec_adjustment; 1124 1125 /* Boot time adjustment, for uptime/monotonic clocks. */ 1126 if (flags & FFCLOCK_UPTIME) 1127 bintime_sub(bt, &ffclock_boottime); 1128 break; 1129#endif 1130 default: 1131 return (EINVAL); 1132 break; 1133 } 1134 1135 return (0); 1136} 1137 1138/* 1139 * Initialize a new timecounter and possibly use it. 1140 */ 1141void 1142tc_init(struct timecounter *tc) 1143{ 1144 u_int u; 1145 struct sysctl_oid *tc_root; 1146 1147 u = tc->tc_frequency / tc->tc_counter_mask; 1148 /* XXX: We need some margin here, 10% is a guess */ 1149 u *= 11; 1150 u /= 10; 1151 if (u > hz && tc->tc_quality >= 0) { 1152 tc->tc_quality = -2000; 1153 if (bootverbose) { 1154 printf("Timecounter \"%s\" frequency %ju Hz", 1155 tc->tc_name, (uintmax_t)tc->tc_frequency); 1156 printf(" -- Insufficient hz, needs at least %u\n", u); 1157 } 1158 } else if (tc->tc_quality >= 0 || bootverbose) { 1159 printf("Timecounter \"%s\" frequency %ju Hz quality %d\n", 1160 tc->tc_name, (uintmax_t)tc->tc_frequency, 1161 tc->tc_quality); 1162 } 1163 1164 tc->tc_next = timecounters; 1165 timecounters = tc; 1166 /* 1167 * Set up sysctl tree for this counter. 1168 */ 1169 tc_root = SYSCTL_ADD_NODE(NULL, 1170 SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name, 1171 CTLFLAG_RW, 0, "timecounter description"); 1172 SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1173 "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0, 1174 "mask for implemented bits"); 1175 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1176 "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc), 1177 sysctl_kern_timecounter_get, "IU", "current timecounter value"); 1178 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1179 "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc), 1180 sysctl_kern_timecounter_freq, "QU", "timecounter frequency"); 1181 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1182 "quality", CTLFLAG_RD, &(tc->tc_quality), 0, 1183 "goodness of time counter"); 1184 /* 1185 * Do not automatically switch if the current tc was specifically 1186 * chosen. Never automatically use a timecounter with negative quality. 1187 * Even though we run on the dummy counter, switching here may be 1188 * worse since this timecounter may not be monotonic. 1189 */ 1190 if (tc_chosen) 1191 return; 1192 if (tc->tc_quality < 0) 1193 return; 1194 if (tc->tc_quality < timecounter->tc_quality) 1195 return; 1196 if (tc->tc_quality == timecounter->tc_quality && 1197 tc->tc_frequency < timecounter->tc_frequency) 1198 return; 1199 (void)tc->tc_get_timecount(tc); 1200 (void)tc->tc_get_timecount(tc); 1201 timecounter = tc; 1202} 1203 1204/* Report the frequency of the current timecounter. */ 1205uint64_t 1206tc_getfrequency(void) 1207{ 1208 1209 return (timehands->th_counter->tc_frequency); 1210} 1211 1212/* 1213 * Step our concept of UTC. This is done by modifying our estimate of 1214 * when we booted. 1215 * XXX: not locked. 1216 */ 1217void 1218tc_setclock(struct timespec *ts) 1219{ 1220 struct timespec tbef, taft; 1221 struct bintime bt, bt2; 1222 1223 cpu_tick_calibrate(1); 1224 nanotime(&tbef); 1225 timespec2bintime(ts, &bt); 1226 binuptime(&bt2); 1227 bintime_sub(&bt, &bt2); 1228 bintime_add(&bt2, &boottimebin); 1229 boottimebin = bt; 1230 bintime2timeval(&bt, &boottime); 1231 1232 /* XXX fiddle all the little crinkly bits around the fiords... */ 1233 tc_windup(); 1234 nanotime(&taft); 1235 if (timestepwarnings) { 1236 log(LOG_INFO, 1237 "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n", 1238 (intmax_t)tbef.tv_sec, tbef.tv_nsec, 1239 (intmax_t)taft.tv_sec, taft.tv_nsec, 1240 (intmax_t)ts->tv_sec, ts->tv_nsec); 1241 } 1242 cpu_tick_calibrate(1); 1243} 1244 1245/* 1246 * Initialize the next struct timehands in the ring and make 1247 * it the active timehands. Along the way we might switch to a different 1248 * timecounter and/or do seconds processing in NTP. Slightly magic. 1249 */ 1250static void 1251tc_windup(void) 1252{ 1253 struct bintime bt; 1254 struct timehands *th, *tho; 1255 uint64_t scale; 1256 u_int delta, ncount, ogen; 1257 int i; 1258 time_t t; 1259 1260 /* 1261 * Make the next timehands a copy of the current one, but do not 1262 * overwrite the generation or next pointer. While we update 1263 * the contents, the generation must be zero. 1264 */ 1265 tho = timehands; 1266 th = tho->th_next; 1267 ogen = th->th_generation; 1268 th->th_generation = 0; 1269 bcopy(tho, th, offsetof(struct timehands, th_generation)); 1270 1271 /* 1272 * Capture a timecounter delta on the current timecounter and if 1273 * changing timecounters, a counter value from the new timecounter. 1274 * Update the offset fields accordingly. 1275 */ 1276 delta = tc_delta(th); 1277 if (th->th_counter != timecounter) 1278 ncount = timecounter->tc_get_timecount(timecounter); 1279 else 1280 ncount = 0; 1281#ifdef FFCLOCK 1282 ffclock_windup(delta); 1283#endif 1284 th->th_offset_count += delta; 1285 th->th_offset_count &= th->th_counter->tc_counter_mask; 1286 while (delta > th->th_counter->tc_frequency) { 1287 /* Eat complete unadjusted seconds. */ 1288 delta -= th->th_counter->tc_frequency; 1289 th->th_offset.sec++; 1290 } 1291 if ((delta > th->th_counter->tc_frequency / 2) && 1292 (th->th_scale * delta < ((uint64_t)1 << 63))) { 1293 /* The product th_scale * delta just barely overflows. */ 1294 th->th_offset.sec++; 1295 } 1296 bintime_addx(&th->th_offset, th->th_scale * delta); 1297 1298 /* 1299 * Hardware latching timecounters may not generate interrupts on 1300 * PPS events, so instead we poll them. There is a finite risk that 1301 * the hardware might capture a count which is later than the one we 1302 * got above, and therefore possibly in the next NTP second which might 1303 * have a different rate than the current NTP second. It doesn't 1304 * matter in practice. 1305 */ 1306 if (tho->th_counter->tc_poll_pps) 1307 tho->th_counter->tc_poll_pps(tho->th_counter); 1308 1309 /* 1310 * Deal with NTP second processing. The for loop normally 1311 * iterates at most once, but in extreme situations it might 1312 * keep NTP sane if timeouts are not run for several seconds. 1313 * At boot, the time step can be large when the TOD hardware 1314 * has been read, so on really large steps, we call 1315 * ntp_update_second only twice. We need to call it twice in 1316 * case we missed a leap second. 1317 */ 1318 bt = th->th_offset; 1319 bintime_add(&bt, &boottimebin); 1320 i = bt.sec - tho->th_microtime.tv_sec; 1321 if (i > LARGE_STEP) 1322 i = 2; 1323 for (; i > 0; i--) { 1324 t = bt.sec; 1325 ntp_update_second(&th->th_adjustment, &bt.sec); 1326 if (bt.sec != t) 1327 boottimebin.sec += bt.sec - t; 1328 } 1329 /* Update the UTC timestamps used by the get*() functions. */ 1330 /* XXX shouldn't do this here. Should force non-`get' versions. */ 1331 bintime2timeval(&bt, &th->th_microtime); 1332 bintime2timespec(&bt, &th->th_nanotime); 1333 1334 /* Now is a good time to change timecounters. */ 1335 if (th->th_counter != timecounter) { 1336#ifndef __arm__ 1337 if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0) 1338 cpu_disable_c2_sleep++; 1339 if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0) 1340 cpu_disable_c2_sleep--; 1341#endif 1342 th->th_counter = timecounter; 1343 th->th_offset_count = ncount; 1344 tc_min_ticktock_freq = max(1, timecounter->tc_frequency / 1345 (((uint64_t)timecounter->tc_counter_mask + 1) / 3)); 1346#ifdef FFCLOCK 1347 ffclock_change_tc(th); 1348#endif 1349 } 1350 1351 /*- 1352 * Recalculate the scaling factor. We want the number of 1/2^64 1353 * fractions of a second per period of the hardware counter, taking 1354 * into account the th_adjustment factor which the NTP PLL/adjtime(2) 1355 * processing provides us with. 1356 * 1357 * The th_adjustment is nanoseconds per second with 32 bit binary 1358 * fraction and we want 64 bit binary fraction of second: 1359 * 1360 * x = a * 2^32 / 10^9 = a * 4.294967296 1361 * 1362 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int 1363 * we can only multiply by about 850 without overflowing, that 1364 * leaves no suitably precise fractions for multiply before divide. 1365 * 1366 * Divide before multiply with a fraction of 2199/512 results in a 1367 * systematic undercompensation of 10PPM of th_adjustment. On a 1368 * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. 1369 * 1370 * We happily sacrifice the lowest of the 64 bits of our result 1371 * to the goddess of code clarity. 1372 * 1373 */ 1374 scale = (uint64_t)1 << 63; 1375 scale += (th->th_adjustment / 1024) * 2199; 1376 scale /= th->th_counter->tc_frequency; 1377 th->th_scale = scale * 2; 1378 1379 /* 1380 * Now that the struct timehands is again consistent, set the new 1381 * generation number, making sure to not make it zero. 1382 */ 1383 if (++ogen == 0) 1384 ogen = 1; 1385 th->th_generation = ogen; 1386 1387 /* Go live with the new struct timehands. */ 1388#ifdef FFCLOCK 1389 switch (sysclock_active) { 1390 case SYSCLOCK_FBCK: 1391#endif 1392 time_second = th->th_microtime.tv_sec; 1393 time_uptime = th->th_offset.sec; 1394#ifdef FFCLOCK 1395 break; 1396 case SYSCLOCK_FFWD: 1397 time_second = fftimehands->tick_time_lerp.sec; 1398 time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec; 1399 break; 1400 } 1401#endif 1402 1403 timehands = th; 1404 timekeep_push_vdso(); 1405} 1406 1407/* Report or change the active timecounter hardware. */ 1408static int 1409sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) 1410{ 1411 char newname[32]; 1412 struct timecounter *newtc, *tc; 1413 int error; 1414 1415 tc = timecounter; 1416 strlcpy(newname, tc->tc_name, sizeof(newname)); 1417 1418 error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req); 1419 if (error != 0 || req->newptr == NULL) 1420 return (error); 1421 /* Record that the tc in use now was specifically chosen. */ 1422 tc_chosen = 1; 1423 if (strcmp(newname, tc->tc_name) == 0) 1424 return (0); 1425 for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) { 1426 if (strcmp(newname, newtc->tc_name) != 0) 1427 continue; 1428 1429 /* Warm up new timecounter. */ 1430 (void)newtc->tc_get_timecount(newtc); 1431 (void)newtc->tc_get_timecount(newtc); 1432 1433 timecounter = newtc; 1434 timekeep_push_vdso(); 1435 return (0); 1436 } 1437 return (EINVAL); 1438} 1439 1440SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW, 1441 0, 0, sysctl_kern_timecounter_hardware, "A", 1442 "Timecounter hardware selected"); 1443 1444 1445/* Report the available timecounter hardware. */ 1446static int 1447sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS) 1448{ 1449 char buf[32], *spc; 1450 struct timecounter *tc; 1451 int error; 1452 1453 spc = ""; 1454 error = 0; 1455 for (tc = timecounters; error == 0 && tc != NULL; tc = tc->tc_next) { 1456 sprintf(buf, "%s%s(%d)", 1457 spc, tc->tc_name, tc->tc_quality); 1458 error = SYSCTL_OUT(req, buf, strlen(buf)); 1459 spc = " "; 1460 } 1461 return (error); 1462} 1463 1464SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD, 1465 0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected"); 1466 1467/* 1468 * RFC 2783 PPS-API implementation. 1469 */ 1470 1471/* 1472 * Return true if the driver is aware of the abi version extensions in the 1473 * pps_state structure, and it supports at least the given abi version number. 1474 */ 1475static inline int 1476abi_aware(struct pps_state *pps, int vers) 1477{ 1478 1479 return ((pps->kcmode & KCMODE_ABIFLAG) && pps->driver_abi >= vers); 1480} 1481 1482static int 1483pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps) 1484{ 1485 int err, timo; 1486 pps_seq_t aseq, cseq; 1487 struct timeval tv; 1488 1489 if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC) 1490 return (EINVAL); 1491 1492 /* 1493 * If no timeout is requested, immediately return whatever values were 1494 * most recently captured. If timeout seconds is -1, that's a request 1495 * to block without a timeout. WITNESS won't let us sleep forever 1496 * without a lock (we really don't need a lock), so just repeatedly 1497 * sleep a long time. 1498 */ 1499 if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) { 1500 if (fapi->timeout.tv_sec == -1) 1501 timo = 0x7fffffff; 1502 else { 1503 tv.tv_sec = fapi->timeout.tv_sec; 1504 tv.tv_usec = fapi->timeout.tv_nsec / 1000; 1505 timo = tvtohz(&tv); 1506 } 1507 aseq = pps->ppsinfo.assert_sequence; 1508 cseq = pps->ppsinfo.clear_sequence; 1509 while (aseq == pps->ppsinfo.assert_sequence && 1510 cseq == pps->ppsinfo.clear_sequence) { 1511 if (abi_aware(pps, 1) && pps->driver_mtx != NULL) { 1512 if (pps->flags & PPSFLAG_MTX_SPIN) { 1513 err = msleep_spin(pps, pps->driver_mtx, 1514 "ppsfch", timo); 1515 } else { 1516 err = msleep(pps, pps->driver_mtx, PCATCH, 1517 "ppsfch", timo); 1518 } 1519 } else { 1520 err = tsleep(pps, PCATCH, "ppsfch", timo); 1521 } 1522 if (err == EWOULDBLOCK) { 1523 if (fapi->timeout.tv_sec == -1) { 1524 continue; 1525 } else { 1526 return (ETIMEDOUT); 1527 } 1528 } else if (err != 0) { 1529 return (err); 1530 } 1531 } 1532 } 1533 1534 pps->ppsinfo.current_mode = pps->ppsparam.mode; 1535 fapi->pps_info_buf = pps->ppsinfo; 1536 1537 return (0); 1538} 1539 1540int 1541pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps) 1542{ 1543 pps_params_t *app; 1544 struct pps_fetch_args *fapi; 1545#ifdef FFCLOCK 1546 struct pps_fetch_ffc_args *fapi_ffc; 1547#endif 1548#ifdef PPS_SYNC 1549 struct pps_kcbind_args *kapi; 1550#endif 1551 1552 KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl")); 1553 switch (cmd) { 1554 case PPS_IOC_CREATE: 1555 return (0); 1556 case PPS_IOC_DESTROY: 1557 return (0); 1558 case PPS_IOC_SETPARAMS: 1559 app = (pps_params_t *)data; 1560 if (app->mode & ~pps->ppscap) 1561 return (EINVAL); 1562#ifdef FFCLOCK 1563 /* Ensure only a single clock is selected for ffc timestamp. */ 1564 if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK) 1565 return (EINVAL); 1566#endif 1567 pps->ppsparam = *app; 1568 return (0); 1569 case PPS_IOC_GETPARAMS: 1570 app = (pps_params_t *)data; 1571 *app = pps->ppsparam; 1572 app->api_version = PPS_API_VERS_1; 1573 return (0); 1574 case PPS_IOC_GETCAP: 1575 *(int*)data = pps->ppscap; 1576 return (0); 1577 case PPS_IOC_FETCH: 1578 fapi = (struct pps_fetch_args *)data; 1579 return (pps_fetch(fapi, pps)); 1580#ifdef FFCLOCK 1581 case PPS_IOC_FETCH_FFCOUNTER: 1582 fapi_ffc = (struct pps_fetch_ffc_args *)data; 1583 if (fapi_ffc->tsformat && fapi_ffc->tsformat != 1584 PPS_TSFMT_TSPEC) 1585 return (EINVAL); 1586 if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec) 1587 return (EOPNOTSUPP); 1588 pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode; 1589 fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc; 1590 /* Overwrite timestamps if feedback clock selected. */ 1591 switch (pps->ppsparam.mode & PPS_TSCLK_MASK) { 1592 case PPS_TSCLK_FBCK: 1593 fapi_ffc->pps_info_buf_ffc.assert_timestamp = 1594 pps->ppsinfo.assert_timestamp; 1595 fapi_ffc->pps_info_buf_ffc.clear_timestamp = 1596 pps->ppsinfo.clear_timestamp; 1597 break; 1598 case PPS_TSCLK_FFWD: 1599 break; 1600 default: 1601 break; 1602 } 1603 return (0); 1604#endif /* FFCLOCK */ 1605 case PPS_IOC_KCBIND: 1606#ifdef PPS_SYNC 1607 kapi = (struct pps_kcbind_args *)data; 1608 /* XXX Only root should be able to do this */ 1609 if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC) 1610 return (EINVAL); 1611 if (kapi->kernel_consumer != PPS_KC_HARDPPS) 1612 return (EINVAL); 1613 if (kapi->edge & ~pps->ppscap) 1614 return (EINVAL); 1615 pps->kcmode = (kapi->edge & KCMODE_EDGEMASK) | 1616 (pps->kcmode & KCMODE_ABIFLAG); 1617 return (0); 1618#else 1619 return (EOPNOTSUPP); 1620#endif 1621 default: 1622 return (ENOIOCTL); 1623 } 1624} 1625 1626void 1627pps_init(struct pps_state *pps) 1628{ 1629 pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT; 1630 if (pps->ppscap & PPS_CAPTUREASSERT) 1631 pps->ppscap |= PPS_OFFSETASSERT; 1632 if (pps->ppscap & PPS_CAPTURECLEAR) 1633 pps->ppscap |= PPS_OFFSETCLEAR; 1634#ifdef FFCLOCK 1635 pps->ppscap |= PPS_TSCLK_MASK; 1636#endif 1637 pps->kcmode &= ~KCMODE_ABIFLAG; 1638} 1639 1640void 1641pps_init_abi(struct pps_state *pps) 1642{ 1643 1644 pps_init(pps); 1645 if (pps->driver_abi > 0) { 1646 pps->kcmode |= KCMODE_ABIFLAG; 1647 pps->kernel_abi = PPS_ABI_VERSION; 1648 } 1649} 1650 1651void 1652pps_capture(struct pps_state *pps) 1653{ 1654 struct timehands *th; 1655 1656 KASSERT(pps != NULL, ("NULL pps pointer in pps_capture")); 1657 th = timehands; 1658 pps->capgen = th->th_generation; 1659 pps->capth = th; 1660#ifdef FFCLOCK 1661 pps->capffth = fftimehands; 1662#endif 1663 pps->capcount = th->th_counter->tc_get_timecount(th->th_counter); 1664 if (pps->capgen != th->th_generation) 1665 pps->capgen = 0; 1666} 1667 1668void 1669pps_event(struct pps_state *pps, int event) 1670{ 1671 struct bintime bt; 1672 struct timespec ts, *tsp, *osp; 1673 u_int tcount, *pcount; 1674 int foff, fhard; 1675 pps_seq_t *pseq; 1676#ifdef FFCLOCK 1677 struct timespec *tsp_ffc; 1678 pps_seq_t *pseq_ffc; 1679 ffcounter *ffcount; 1680#endif 1681 1682 KASSERT(pps != NULL, ("NULL pps pointer in pps_event")); 1683 /* Nothing to do if not currently set to capture this event type. */ 1684 if ((event & pps->ppsparam.mode) == 0) 1685 return; 1686 /* If the timecounter was wound up underneath us, bail out. */ 1687 if (pps->capgen == 0 || pps->capgen != pps->capth->th_generation) 1688 return; 1689 1690 /* Things would be easier with arrays. */ 1691 if (event == PPS_CAPTUREASSERT) { 1692 tsp = &pps->ppsinfo.assert_timestamp; 1693 osp = &pps->ppsparam.assert_offset; 1694 foff = pps->ppsparam.mode & PPS_OFFSETASSERT; 1695 fhard = pps->kcmode & PPS_CAPTUREASSERT; 1696 pcount = &pps->ppscount[0]; 1697 pseq = &pps->ppsinfo.assert_sequence; 1698#ifdef FFCLOCK 1699 ffcount = &pps->ppsinfo_ffc.assert_ffcount; 1700 tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp; 1701 pseq_ffc = &pps->ppsinfo_ffc.assert_sequence; 1702#endif 1703 } else { 1704 tsp = &pps->ppsinfo.clear_timestamp; 1705 osp = &pps->ppsparam.clear_offset; 1706 foff = pps->ppsparam.mode & PPS_OFFSETCLEAR; 1707 fhard = pps->kcmode & PPS_CAPTURECLEAR; 1708 pcount = &pps->ppscount[1]; 1709 pseq = &pps->ppsinfo.clear_sequence; 1710#ifdef FFCLOCK 1711 ffcount = &pps->ppsinfo_ffc.clear_ffcount; 1712 tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp; 1713 pseq_ffc = &pps->ppsinfo_ffc.clear_sequence; 1714#endif 1715 } 1716 1717 /* 1718 * If the timecounter changed, we cannot compare the count values, so 1719 * we have to drop the rest of the PPS-stuff until the next event. 1720 */ 1721 if (pps->ppstc != pps->capth->th_counter) { 1722 pps->ppstc = pps->capth->th_counter; 1723 *pcount = pps->capcount; 1724 pps->ppscount[2] = pps->capcount; 1725 return; 1726 } 1727 1728 /* Convert the count to a timespec. */ 1729 tcount = pps->capcount - pps->capth->th_offset_count; 1730 tcount &= pps->capth->th_counter->tc_counter_mask; 1731 bt = pps->capth->th_offset; 1732 bintime_addx(&bt, pps->capth->th_scale * tcount); 1733 bintime_add(&bt, &boottimebin); 1734 bintime2timespec(&bt, &ts); 1735 1736 /* If the timecounter was wound up underneath us, bail out. */ 1737 if (pps->capgen != pps->capth->th_generation) 1738 return; 1739 1740 *pcount = pps->capcount; 1741 (*pseq)++; 1742 *tsp = ts; 1743 1744 if (foff) { 1745 timespecadd(tsp, osp); 1746 if (tsp->tv_nsec < 0) { 1747 tsp->tv_nsec += 1000000000; 1748 tsp->tv_sec -= 1; 1749 } 1750 } 1751 1752#ifdef FFCLOCK 1753 *ffcount = pps->capffth->tick_ffcount + tcount; 1754 bt = pps->capffth->tick_time; 1755 ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt); 1756 bintime_add(&bt, &pps->capffth->tick_time); 1757 bintime2timespec(&bt, &ts); 1758 (*pseq_ffc)++; 1759 *tsp_ffc = ts; 1760#endif 1761 1762#ifdef PPS_SYNC 1763 if (fhard) { 1764 uint64_t scale; 1765 1766 /* 1767 * Feed the NTP PLL/FLL. 1768 * The FLL wants to know how many (hardware) nanoseconds 1769 * elapsed since the previous event. 1770 */ 1771 tcount = pps->capcount - pps->ppscount[2]; 1772 pps->ppscount[2] = pps->capcount; 1773 tcount &= pps->capth->th_counter->tc_counter_mask; 1774 scale = (uint64_t)1 << 63; 1775 scale /= pps->capth->th_counter->tc_frequency; 1776 scale *= 2; 1777 bt.sec = 0; 1778 bt.frac = 0; 1779 bintime_addx(&bt, scale * tcount); 1780 bintime2timespec(&bt, &ts); 1781 hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec); 1782 } 1783#endif 1784 1785 /* Wakeup anyone sleeping in pps_fetch(). */ 1786 wakeup(pps); 1787} 1788 1789/* 1790 * Timecounters need to be updated every so often to prevent the hardware 1791 * counter from overflowing. Updating also recalculates the cached values 1792 * used by the get*() family of functions, so their precision depends on 1793 * the update frequency. 1794 */ 1795 1796static int tc_tick; 1797SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0, 1798 "Approximate number of hardclock ticks in a millisecond"); 1799 1800void 1801tc_ticktock(int cnt) 1802{ 1803 static int count; 1804 1805 count += cnt; 1806 if (count < tc_tick) 1807 return; 1808 count = 0; 1809 tc_windup(); 1810} 1811 1812static void __inline 1813tc_adjprecision(void) 1814{ 1815 int t; 1816 1817 if (tc_timepercentage > 0) { 1818 t = (99 + tc_timepercentage) / tc_timepercentage; 1819 tc_precexp = fls(t + (t >> 1)) - 1; 1820 FREQ2BT(hz / tc_tick, &bt_timethreshold); 1821 FREQ2BT(hz, &bt_tickthreshold); 1822 bintime_shift(&bt_timethreshold, tc_precexp); 1823 bintime_shift(&bt_tickthreshold, tc_precexp); 1824 } else { 1825 tc_precexp = 31; 1826 bt_timethreshold.sec = INT_MAX; 1827 bt_timethreshold.frac = ~(uint64_t)0; 1828 bt_tickthreshold = bt_timethreshold; 1829 } 1830 sbt_timethreshold = bttosbt(bt_timethreshold); 1831 sbt_tickthreshold = bttosbt(bt_tickthreshold); 1832} 1833 1834static int 1835sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS) 1836{ 1837 int error, val; 1838 1839 val = tc_timepercentage; 1840 error = sysctl_handle_int(oidp, &val, 0, req); 1841 if (error != 0 || req->newptr == NULL) 1842 return (error); 1843 tc_timepercentage = val; 1844 tc_adjprecision(); 1845 return (0); 1846} 1847 1848static void 1849inittimecounter(void *dummy) 1850{ 1851 u_int p; 1852 int tick_rate; 1853 1854 /* 1855 * Set the initial timeout to 1856 * max(1, <approx. number of hardclock ticks in a millisecond>). 1857 * People should probably not use the sysctl to set the timeout 1858 * to smaller than its initial value, since that value is the 1859 * smallest reasonable one. If they want better timestamps they 1860 * should use the non-"get"* functions. 1861 */ 1862 if (hz > 1000) 1863 tc_tick = (hz + 500) / 1000; 1864 else 1865 tc_tick = 1; 1866 tc_adjprecision(); 1867 FREQ2BT(hz, &tick_bt); 1868 tick_sbt = bttosbt(tick_bt); 1869 tick_rate = hz / tc_tick; 1870 FREQ2BT(tick_rate, &tc_tick_bt); 1871 tc_tick_sbt = bttosbt(tc_tick_bt); 1872 p = (tc_tick * 1000000) / hz; 1873 printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000); 1874 1875#ifdef FFCLOCK 1876 ffclock_init(); 1877#endif 1878 /* warm up new timecounter (again) and get rolling. */ 1879 (void)timecounter->tc_get_timecount(timecounter); 1880 (void)timecounter->tc_get_timecount(timecounter); 1881 tc_windup(); 1882} 1883 1884SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL); 1885 1886/* Cpu tick handling -------------------------------------------------*/ 1887 1888static int cpu_tick_variable; 1889static uint64_t cpu_tick_frequency; 1890 1891static DPCPU_DEFINE(uint64_t, tc_cpu_ticks_base); 1892static DPCPU_DEFINE(unsigned, tc_cpu_ticks_last); 1893 1894static uint64_t 1895tc_cpu_ticks(void) 1896{ 1897 struct timecounter *tc; 1898 uint64_t res, *base; 1899 unsigned u, *last; 1900 1901 critical_enter(); 1902 base = DPCPU_PTR(tc_cpu_ticks_base); 1903 last = DPCPU_PTR(tc_cpu_ticks_last); 1904 tc = timehands->th_counter; 1905 u = tc->tc_get_timecount(tc) & tc->tc_counter_mask; 1906 if (u < *last) 1907 *base += (uint64_t)tc->tc_counter_mask + 1; 1908 *last = u; 1909 res = u + *base; 1910 critical_exit(); 1911 return (res); 1912} 1913 1914void 1915cpu_tick_calibration(void) 1916{ 1917 static time_t last_calib; 1918 1919 if (time_uptime != last_calib && !(time_uptime & 0xf)) { 1920 cpu_tick_calibrate(0); 1921 last_calib = time_uptime; 1922 } 1923} 1924 1925/* 1926 * This function gets called every 16 seconds on only one designated 1927 * CPU in the system from hardclock() via cpu_tick_calibration()(). 1928 * 1929 * Whenever the real time clock is stepped we get called with reset=1 1930 * to make sure we handle suspend/resume and similar events correctly. 1931 */ 1932 1933static void 1934cpu_tick_calibrate(int reset) 1935{ 1936 static uint64_t c_last; 1937 uint64_t c_this, c_delta; 1938 static struct bintime t_last; 1939 struct bintime t_this, t_delta; 1940 uint32_t divi; 1941 1942 if (reset) { 1943 /* The clock was stepped, abort & reset */ 1944 t_last.sec = 0; 1945 return; 1946 } 1947 1948 /* we don't calibrate fixed rate cputicks */ 1949 if (!cpu_tick_variable) 1950 return; 1951 1952 getbinuptime(&t_this); 1953 c_this = cpu_ticks(); 1954 if (t_last.sec != 0) { 1955 c_delta = c_this - c_last; 1956 t_delta = t_this; 1957 bintime_sub(&t_delta, &t_last); 1958 /* 1959 * Headroom: 1960 * 2^(64-20) / 16[s] = 1961 * 2^(44) / 16[s] = 1962 * 17.592.186.044.416 / 16 = 1963 * 1.099.511.627.776 [Hz] 1964 */ 1965 divi = t_delta.sec << 20; 1966 divi |= t_delta.frac >> (64 - 20); 1967 c_delta <<= 20; 1968 c_delta /= divi; 1969 if (c_delta > cpu_tick_frequency) { 1970 if (0 && bootverbose) 1971 printf("cpu_tick increased to %ju Hz\n", 1972 c_delta); 1973 cpu_tick_frequency = c_delta; 1974 } 1975 } 1976 c_last = c_this; 1977 t_last = t_this; 1978} 1979 1980void 1981set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var) 1982{ 1983 1984 if (func == NULL) { 1985 cpu_ticks = tc_cpu_ticks; 1986 } else { 1987 cpu_tick_frequency = freq; 1988 cpu_tick_variable = var; 1989 cpu_ticks = func; 1990 } 1991} 1992 1993uint64_t 1994cpu_tickrate(void) 1995{ 1996 1997 if (cpu_ticks == tc_cpu_ticks) 1998 return (tc_getfrequency()); 1999 return (cpu_tick_frequency); 2000} 2001 2002/* 2003 * We need to be slightly careful converting cputicks to microseconds. 2004 * There is plenty of margin in 64 bits of microseconds (half a million 2005 * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply 2006 * before divide conversion (to retain precision) we find that the 2007 * margin shrinks to 1.5 hours (one millionth of 146y). 2008 * With a three prong approach we never lose significant bits, no 2009 * matter what the cputick rate and length of timeinterval is. 2010 */ 2011 2012uint64_t 2013cputick2usec(uint64_t tick) 2014{ 2015 2016 if (tick > 18446744073709551LL) /* floor(2^64 / 1000) */ 2017 return (tick / (cpu_tickrate() / 1000000LL)); 2018 else if (tick > 18446744073709LL) /* floor(2^64 / 1000000) */ 2019 return ((tick * 1000LL) / (cpu_tickrate() / 1000LL)); 2020 else 2021 return ((tick * 1000000LL) / cpu_tickrate()); 2022} 2023 2024cpu_tick_f *cpu_ticks = tc_cpu_ticks; 2025 2026static int vdso_th_enable = 1; 2027static int 2028sysctl_fast_gettime(SYSCTL_HANDLER_ARGS) 2029{ 2030 int old_vdso_th_enable, error; 2031 2032 old_vdso_th_enable = vdso_th_enable; 2033 error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req); 2034 if (error != 0) 2035 return (error); 2036 vdso_th_enable = old_vdso_th_enable; 2037 timekeep_push_vdso(); 2038 return (0); 2039} 2040SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime, 2041 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 2042 NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day"); 2043 2044uint32_t 2045tc_fill_vdso_timehands(struct vdso_timehands *vdso_th) 2046{ 2047 struct timehands *th; 2048 uint32_t enabled; 2049 2050 th = timehands; 2051 vdso_th->th_algo = VDSO_TH_ALGO_1; 2052 vdso_th->th_scale = th->th_scale; 2053 vdso_th->th_offset_count = th->th_offset_count; 2054 vdso_th->th_counter_mask = th->th_counter->tc_counter_mask; 2055 vdso_th->th_offset = th->th_offset; 2056 vdso_th->th_boottime = boottimebin; 2057 enabled = cpu_fill_vdso_timehands(vdso_th); 2058 if (!vdso_th_enable) 2059 enabled = 0; 2060 return (enabled); 2061} 2062 2063#ifdef COMPAT_FREEBSD32 2064uint32_t 2065tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) 2066{ 2067 struct timehands *th; 2068 uint32_t enabled; 2069 2070 th = timehands; 2071 vdso_th32->th_algo = VDSO_TH_ALGO_1; 2072 *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale; 2073 vdso_th32->th_offset_count = th->th_offset_count; 2074 vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask; 2075 vdso_th32->th_offset.sec = th->th_offset.sec; 2076 *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac; 2077 vdso_th32->th_boottime.sec = boottimebin.sec; 2078 *(uint64_t *)&vdso_th32->th_boottime.frac[0] = boottimebin.frac; 2079 enabled = cpu_fill_vdso_timehands32(vdso_th32); 2080 if (!vdso_th_enable) 2081 enabled = 0; 2082 return (enabled); 2083} 2084#endif 2085