1/*-
2 * ----------------------------------------------------------------------------
3 * "THE BEER-WARE LICENSE" (Revision 42):
4 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you
5 * can do whatever you want with this stuff. If we meet some day, and you think
6 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
7 * ----------------------------------------------------------------------------
8 *
9 * Copyright (c) 2011 The FreeBSD Foundation
10 * All rights reserved.
11 *
12 * Portions of this software were developed by Julien Ridoux at the University
13 * of Melbourne under sponsorship from the FreeBSD Foundation.
14 */
15
16#include <sys/cdefs.h>
17__FBSDID("$FreeBSD: stable/11/sys/kern/kern_tc.c 304974 2016-08-29 04:45:58Z kib $");
18
19#include "opt_compat.h"
20#include "opt_ntp.h"
21#include "opt_ffclock.h"
22
23#include <sys/param.h>
24#include <sys/kernel.h>
25#include <sys/limits.h>
26#include <sys/lock.h>
27#include <sys/mutex.h>
28#include <sys/sbuf.h>
29#include <sys/sysctl.h>
30#include <sys/syslog.h>
31#include <sys/systm.h>
32#include <sys/timeffc.h>
33#include <sys/timepps.h>
34#include <sys/timetc.h>
35#include <sys/timex.h>
36#include <sys/vdso.h>
37
38/*
39 * A large step happens on boot. This constant detects such steps.
40 * It is relatively small so that ntp_update_second gets called enough
41 * in the typical 'missed a couple of seconds' case, but doesn't loop
42 * forever when the time step is large.
43 */
44#define LARGE_STEP 200
45
46/*
47 * Implement a dummy timecounter which we can use until we get a real one
48 * in the air. This allows the console and other early stuff to use
49 * time services.
50 */
51
52static u_int
53dummy_get_timecount(struct timecounter *tc)
54{
55 static u_int now;
56
57 return (++now);
58}
59
60static struct timecounter dummy_timecounter = {
61 dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000
62};
63
64struct timehands {
65 /* These fields must be initialized by the driver. */
66 struct timecounter *th_counter;
67 int64_t th_adjustment;
68 uint64_t th_scale;
69 u_int th_offset_count;
70 struct bintime th_offset;
71 struct bintime th_bintime;
72 struct timeval th_microtime;
73 struct timespec th_nanotime;
74 struct bintime th_boottime;
75 /* Fields not to be copied in tc_windup start with th_generation. */
76 u_int th_generation;
77 struct timehands *th_next;
78};
79
80static struct timehands th0;
81static struct timehands th1 = {
82 .th_next = &th0
83};
84static struct timehands th0 = {
85 .th_counter = &dummy_timecounter,
86 .th_scale = (uint64_t)-1 / 1000000,
87 .th_offset = { .sec = 1 },
88 .th_generation = 1,
89 .th_next = &th1
90};
91
92static struct timehands *volatile timehands = &th0;
93struct timecounter *timecounter = &dummy_timecounter;
94static struct timecounter *timecounters = &dummy_timecounter;
95
96int tc_min_ticktock_freq = 1;
97
98volatile time_t time_second = 1;
99volatile time_t time_uptime = 1;
100
101struct bintime boottimebin;
102struct timeval boottime;
103static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS);
104SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD,
105 NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime");
106
107SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, "");
108static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, "");
109
110static int timestepwarnings;
111SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW,
112 &timestepwarnings, 0, "Log time steps");
113
114struct bintime bt_timethreshold;
115struct bintime bt_tickthreshold;
116sbintime_t sbt_timethreshold;
117sbintime_t sbt_tickthreshold;
118struct bintime tc_tick_bt;
119sbintime_t tc_tick_sbt;
120int tc_precexp;
121int tc_timepercentage = TC_DEFAULTPERC;
122static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS);
123SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation,
124 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 0, 0,
125 sysctl_kern_timecounter_adjprecision, "I",
126 "Allowed time interval deviation in percents");
127
128static int tc_chosen; /* Non-zero if a specific tc was chosen via sysctl. */
129
130static void tc_windup(struct bintime *new_boottimebin);
131static void cpu_tick_calibrate(int);
132
133void dtrace_getnanotime(struct timespec *tsp);
134
135static int
136sysctl_kern_boottime(SYSCTL_HANDLER_ARGS)
137{
138 struct timeval boottime_x;
139
140 getboottime(&boottime_x);
141
142#ifndef __mips__
143#ifdef SCTL_MASK32
144 int tv[2];
145
146 if (req->flags & SCTL_MASK32) {
147 tv[0] = boottime_x.tv_sec;
148 tv[1] = boottime_x.tv_usec;
149 return (SYSCTL_OUT(req, tv, sizeof(tv)));
150 }
151#endif
152#endif
153 return (SYSCTL_OUT(req, &boottime_x, sizeof(boottime_x)));
154}
155
156static int
157sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS)
158{
159 u_int ncount;
160 struct timecounter *tc = arg1;
161
162 ncount = tc->tc_get_timecount(tc);
163 return (sysctl_handle_int(oidp, &ncount, 0, req));
164}
165
166static int
167sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS)
168{
169 uint64_t freq;
170 struct timecounter *tc = arg1;
171
172 freq = tc->tc_frequency;
173 return (sysctl_handle_64(oidp, &freq, 0, req));
174}
175
176/*
177 * Return the difference between the timehands' counter value now and what
178 * was when we copied it to the timehands' offset_count.
179 */
180static __inline u_int
181tc_delta(struct timehands *th)
182{
183 struct timecounter *tc;
184
185 tc = th->th_counter;
186 return ((tc->tc_get_timecount(tc) - th->th_offset_count) &
187 tc->tc_counter_mask);
188}
189
190/*
191 * Functions for reading the time. We have to loop until we are sure that
192 * the timehands that we operated on was not updated under our feet. See
193 * the comment in <sys/time.h> for a description of these 12 functions.
194 */
195
196#ifdef FFCLOCK
197void
198fbclock_binuptime(struct bintime *bt)
199{
200 struct timehands *th;
201 unsigned int gen;
202
203 do {
204 th = timehands;
205 gen = atomic_load_acq_int(&th->th_generation);
206 *bt = th->th_offset;
207 bintime_addx(bt, th->th_scale * tc_delta(th));
208 atomic_thread_fence_acq();
209 } while (gen == 0 || gen != th->th_generation);
210}
211
212void
213fbclock_nanouptime(struct timespec *tsp)
214{
215 struct bintime bt;
216
217 fbclock_binuptime(&bt);
218 bintime2timespec(&bt, tsp);
219}
220
221void
222fbclock_microuptime(struct timeval *tvp)
223{
224 struct bintime bt;
225
226 fbclock_binuptime(&bt);
227 bintime2timeval(&bt, tvp);
228}
229
230void
231fbclock_bintime(struct bintime *bt)
232{
233 struct timehands *th;
234 unsigned int gen;
235
236 do {
237 th = timehands;
238 gen = atomic_load_acq_int(&th->th_generation);
239 *bt = th->th_bintime;
240 bintime_addx(bt, th->th_scale * tc_delta(th));
241 atomic_thread_fence_acq();
242 } while (gen == 0 || gen != th->th_generation);
243}
244
245void
246fbclock_nanotime(struct timespec *tsp)
247{
248 struct bintime bt;
249
250 fbclock_bintime(&bt);
251 bintime2timespec(&bt, tsp);
252}
253
254void
255fbclock_microtime(struct timeval *tvp)
256{
257 struct bintime bt;
258
259 fbclock_bintime(&bt);
260 bintime2timeval(&bt, tvp);
261}
262
263void
264fbclock_getbinuptime(struct bintime *bt)
265{
266 struct timehands *th;
267 unsigned int gen;
268
269 do {
270 th = timehands;
271 gen = atomic_load_acq_int(&th->th_generation);
272 *bt = th->th_offset;
273 atomic_thread_fence_acq();
274 } while (gen == 0 || gen != th->th_generation);
275}
276
277void
278fbclock_getnanouptime(struct timespec *tsp)
279{
280 struct timehands *th;
281 unsigned int gen;
282
283 do {
284 th = timehands;
285 gen = atomic_load_acq_int(&th->th_generation);
286 bintime2timespec(&th->th_offset, tsp);
287 atomic_thread_fence_acq();
288 } while (gen == 0 || gen != th->th_generation);
289}
290
291void
292fbclock_getmicrouptime(struct timeval *tvp)
293{
294 struct timehands *th;
295 unsigned int gen;
296
297 do {
298 th = timehands;
299 gen = atomic_load_acq_int(&th->th_generation);
300 bintime2timeval(&th->th_offset, tvp);
301 atomic_thread_fence_acq();
302 } while (gen == 0 || gen != th->th_generation);
303}
304
305void
306fbclock_getbintime(struct bintime *bt)
307{
308 struct timehands *th;
309 unsigned int gen;
310
311 do {
312 th = timehands;
313 gen = atomic_load_acq_int(&th->th_generation);
314 *bt = th->th_bintime;
315 atomic_thread_fence_acq();
316 } while (gen == 0 || gen != th->th_generation);
317}
318
319void
320fbclock_getnanotime(struct timespec *tsp)
321{
322 struct timehands *th;
323 unsigned int gen;
324
325 do {
326 th = timehands;
327 gen = atomic_load_acq_int(&th->th_generation);
328 *tsp = th->th_nanotime;
329 atomic_thread_fence_acq();
330 } while (gen == 0 || gen != th->th_generation);
331}
332
333void
334fbclock_getmicrotime(struct timeval *tvp)
335{
336 struct timehands *th;
337 unsigned int gen;
338
339 do {
340 th = timehands;
341 gen = atomic_load_acq_int(&th->th_generation);
342 *tvp = th->th_microtime;
343 atomic_thread_fence_acq();
344 } while (gen == 0 || gen != th->th_generation);
345}
346#else /* !FFCLOCK */
347void
348binuptime(struct bintime *bt)
349{
350 struct timehands *th;
351 u_int gen;
352
353 do {
354 th = timehands;
355 gen = atomic_load_acq_int(&th->th_generation);
356 *bt = th->th_offset;
357 bintime_addx(bt, th->th_scale * tc_delta(th));
358 atomic_thread_fence_acq();
359 } while (gen == 0 || gen != th->th_generation);
360}
361
362void
363nanouptime(struct timespec *tsp)
364{
365 struct bintime bt;
366
367 binuptime(&bt);
368 bintime2timespec(&bt, tsp);
369}
370
371void
372microuptime(struct timeval *tvp)
373{
374 struct bintime bt;
375
376 binuptime(&bt);
377 bintime2timeval(&bt, tvp);
378}
379
380void
381bintime(struct bintime *bt)
382{
383 struct timehands *th;
384 u_int gen;
385
386 do {
387 th = timehands;
388 gen = atomic_load_acq_int(&th->th_generation);
389 *bt = th->th_bintime;
390 bintime_addx(bt, th->th_scale * tc_delta(th));
391 atomic_thread_fence_acq();
392 } while (gen == 0 || gen != th->th_generation);
393}
394
395void
396nanotime(struct timespec *tsp)
397{
398 struct bintime bt;
399
400 bintime(&bt);
401 bintime2timespec(&bt, tsp);
402}
403
404void
405microtime(struct timeval *tvp)
406{
407 struct bintime bt;
408
409 bintime(&bt);
410 bintime2timeval(&bt, tvp);
411}
412
413void
414getbinuptime(struct bintime *bt)
415{
416 struct timehands *th;
417 u_int gen;
418
419 do {
420 th = timehands;
421 gen = atomic_load_acq_int(&th->th_generation);
422 *bt = th->th_offset;
423 atomic_thread_fence_acq();
424 } while (gen == 0 || gen != th->th_generation);
425}
426
427void
428getnanouptime(struct timespec *tsp)
429{
430 struct timehands *th;
431 u_int gen;
432
433 do {
434 th = timehands;
435 gen = atomic_load_acq_int(&th->th_generation);
436 bintime2timespec(&th->th_offset, tsp);
437 atomic_thread_fence_acq();
438 } while (gen == 0 || gen != th->th_generation);
439}
440
441void
442getmicrouptime(struct timeval *tvp)
443{
444 struct timehands *th;
445 u_int gen;
446
447 do {
448 th = timehands;
449 gen = atomic_load_acq_int(&th->th_generation);
450 bintime2timeval(&th->th_offset, tvp);
451 atomic_thread_fence_acq();
452 } while (gen == 0 || gen != th->th_generation);
453}
454
455void
456getbintime(struct bintime *bt)
457{
458 struct timehands *th;
459 u_int gen;
460
461 do {
462 th = timehands;
463 gen = atomic_load_acq_int(&th->th_generation);
464 *bt = th->th_bintime;
465 atomic_thread_fence_acq();
466 } while (gen == 0 || gen != th->th_generation);
467}
468
469void
470getnanotime(struct timespec *tsp)
471{
472 struct timehands *th;
473 u_int gen;
474
475 do {
476 th = timehands;
477 gen = atomic_load_acq_int(&th->th_generation);
478 *tsp = th->th_nanotime;
479 atomic_thread_fence_acq();
480 } while (gen == 0 || gen != th->th_generation);
481}
482
483void
484getmicrotime(struct timeval *tvp)
485{
486 struct timehands *th;
487 u_int gen;
488
489 do {
490 th = timehands;
491 gen = atomic_load_acq_int(&th->th_generation);
492 *tvp = th->th_microtime;
493 atomic_thread_fence_acq();
494 } while (gen == 0 || gen != th->th_generation);
495}
496#endif /* FFCLOCK */
497
498void
499getboottime(struct timeval *boottime_x)
500{
501 struct bintime boottimebin_x;
502
503 getboottimebin(&boottimebin_x);
504 bintime2timeval(&boottimebin_x, boottime_x);
505}
506
507void
508getboottimebin(struct bintime *boottimebin_x)
509{
510 struct timehands *th;
511 u_int gen;
512
513 do {
514 th = timehands;
515 gen = atomic_load_acq_int(&th->th_generation);
516 *boottimebin_x = th->th_boottime;
517 atomic_thread_fence_acq();
518 } while (gen == 0 || gen != th->th_generation);
519}
520
521#ifdef FFCLOCK
522/*
523 * Support for feed-forward synchronization algorithms. This is heavily inspired
524 * by the timehands mechanism but kept independent from it. *_windup() functions
525 * have some connection to avoid accessing the timecounter hardware more than
526 * necessary.
527 */
528
529/* Feed-forward clock estimates kept updated by the synchronization daemon. */
530struct ffclock_estimate ffclock_estimate;
531struct bintime ffclock_boottime; /* Feed-forward boot time estimate. */
532uint32_t ffclock_status; /* Feed-forward clock status. */
533int8_t ffclock_updated; /* New estimates are available. */
534struct mtx ffclock_mtx; /* Mutex on ffclock_estimate. */
535
536struct fftimehands {
537 struct ffclock_estimate cest;
538 struct bintime tick_time;
539 struct bintime tick_time_lerp;
540 ffcounter tick_ffcount;
541 uint64_t period_lerp;
542 volatile uint8_t gen;
543 struct fftimehands *next;
544};
545
546#define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
547
548static struct fftimehands ffth[10];
549static struct fftimehands *volatile fftimehands = ffth;
550
551static void
552ffclock_init(void)
553{
554 struct fftimehands *cur;
555 struct fftimehands *last;
556
557 memset(ffth, 0, sizeof(ffth));
558
559 last = ffth + NUM_ELEMENTS(ffth) - 1;
560 for (cur = ffth; cur < last; cur++)
561 cur->next = cur + 1;
562 last->next = ffth;
563
564 ffclock_updated = 0;
565 ffclock_status = FFCLOCK_STA_UNSYNC;
566 mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF);
567}
568
569/*
570 * Reset the feed-forward clock estimates. Called from inittodr() to get things
571 * kick started and uses the timecounter nominal frequency as a first period
572 * estimate. Note: this function may be called several time just after boot.
573 * Note: this is the only function that sets the value of boot time for the
574 * monotonic (i.e. uptime) version of the feed-forward clock.
575 */
576void
577ffclock_reset_clock(struct timespec *ts)
578{
579 struct timecounter *tc;
580 struct ffclock_estimate cest;
581
582 tc = timehands->th_counter;
583 memset(&cest, 0, sizeof(struct ffclock_estimate));
584
585 timespec2bintime(ts, &ffclock_boottime);
586 timespec2bintime(ts, &(cest.update_time));
587 ffclock_read_counter(&cest.update_ffcount);
588 cest.leapsec_next = 0;
589 cest.period = ((1ULL << 63) / tc->tc_frequency) << 1;
590 cest.errb_abs = 0;
591 cest.errb_rate = 0;
592 cest.status = FFCLOCK_STA_UNSYNC;
593 cest.leapsec_total = 0;
594 cest.leapsec = 0;
595
596 mtx_lock(&ffclock_mtx);
597 bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate));
598 ffclock_updated = INT8_MAX;
599 mtx_unlock(&ffclock_mtx);
600
601 printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name,
602 (unsigned long long)tc->tc_frequency, (long)ts->tv_sec,
603 (unsigned long)ts->tv_nsec);
604}
605
606/*
607 * Sub-routine to convert a time interval measured in RAW counter units to time
608 * in seconds stored in bintime format.
609 * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be
610 * larger than the max value of u_int (on 32 bit architecture). Loop to consume
611 * extra cycles.
612 */
613static void
614ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt)
615{
616 struct bintime bt2;
617 ffcounter delta, delta_max;
618
619 delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1;
620 bintime_clear(bt);
621 do {
622 if (ffdelta > delta_max)
623 delta = delta_max;
624 else
625 delta = ffdelta;
626 bt2.sec = 0;
627 bt2.frac = period;
628 bintime_mul(&bt2, (unsigned int)delta);
629 bintime_add(bt, &bt2);
630 ffdelta -= delta;
631 } while (ffdelta > 0);
632}
633
634/*
635 * Update the fftimehands.
636 * Push the tick ffcount and time(s) forward based on current clock estimate.
637 * The conversion from ffcounter to bintime relies on the difference clock
638 * principle, whose accuracy relies on computing small time intervals. If a new
639 * clock estimate has been passed by the synchronisation daemon, make it
640 * current, and compute the linear interpolation for monotonic time if needed.
641 */
642static void
643ffclock_windup(unsigned int delta)
644{
645 struct ffclock_estimate *cest;
646 struct fftimehands *ffth;
647 struct bintime bt, gap_lerp;
648 ffcounter ffdelta;
649 uint64_t frac;
650 unsigned int polling;
651 uint8_t forward_jump, ogen;
652
653 /*
654 * Pick the next timehand, copy current ffclock estimates and move tick
655 * times and counter forward.
656 */
657 forward_jump = 0;
658 ffth = fftimehands->next;
659 ogen = ffth->gen;
660 ffth->gen = 0;
661 cest = &ffth->cest;
662 bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate));
663 ffdelta = (ffcounter)delta;
664 ffth->period_lerp = fftimehands->period_lerp;
665
666 ffth->tick_time = fftimehands->tick_time;
667 ffclock_convert_delta(ffdelta, cest->period, &bt);
668 bintime_add(&ffth->tick_time, &bt);
669
670 ffth->tick_time_lerp = fftimehands->tick_time_lerp;
671 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt);
672 bintime_add(&ffth->tick_time_lerp, &bt);
673
674 ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta;
675
676 /*
677 * Assess the status of the clock, if the last update is too old, it is
678 * likely the synchronisation daemon is dead and the clock is free
679 * running.
680 */
681 if (ffclock_updated == 0) {
682 ffdelta = ffth->tick_ffcount - cest->update_ffcount;
683 ffclock_convert_delta(ffdelta, cest->period, &bt);
684 if (bt.sec > 2 * FFCLOCK_SKM_SCALE)
685 ffclock_status |= FFCLOCK_STA_UNSYNC;
686 }
687
688 /*
689 * If available, grab updated clock estimates and make them current.
690 * Recompute time at this tick using the updated estimates. The clock
691 * estimates passed the feed-forward synchronisation daemon may result
692 * in time conversion that is not monotonically increasing (just after
693 * the update). time_lerp is a particular linear interpolation over the
694 * synchronisation algo polling period that ensures monotonicity for the
695 * clock ids requesting it.
696 */
697 if (ffclock_updated > 0) {
698 bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate));
699 ffdelta = ffth->tick_ffcount - cest->update_ffcount;
700 ffth->tick_time = cest->update_time;
701 ffclock_convert_delta(ffdelta, cest->period, &bt);
702 bintime_add(&ffth->tick_time, &bt);
703
704 /* ffclock_reset sets ffclock_updated to INT8_MAX */
705 if (ffclock_updated == INT8_MAX)
706 ffth->tick_time_lerp = ffth->tick_time;
707
708 if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >))
709 forward_jump = 1;
710 else
711 forward_jump = 0;
712
713 bintime_clear(&gap_lerp);
714 if (forward_jump) {
715 gap_lerp = ffth->tick_time;
716 bintime_sub(&gap_lerp, &ffth->tick_time_lerp);
717 } else {
718 gap_lerp = ffth->tick_time_lerp;
719 bintime_sub(&gap_lerp, &ffth->tick_time);
720 }
721
722 /*
723 * The reset from the RTC clock may be far from accurate, and
724 * reducing the gap between real time and interpolated time
725 * could take a very long time if the interpolated clock insists
726 * on strict monotonicity. The clock is reset under very strict
727 * conditions (kernel time is known to be wrong and
728 * synchronization daemon has been restarted recently.
729 * ffclock_boottime absorbs the jump to ensure boot time is
730 * correct and uptime functions stay consistent.
731 */
732 if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) &&
733 ((cest->status & FFCLOCK_STA_UNSYNC) == 0) &&
734 ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) {
735 if (forward_jump)
736 bintime_add(&ffclock_boottime, &gap_lerp);
737 else
738 bintime_sub(&ffclock_boottime, &gap_lerp);
739 ffth->tick_time_lerp = ffth->tick_time;
740 bintime_clear(&gap_lerp);
741 }
742
743 ffclock_status = cest->status;
744 ffth->period_lerp = cest->period;
745
746 /*
747 * Compute corrected period used for the linear interpolation of
748 * time. The rate of linear interpolation is capped to 5000PPM
749 * (5ms/s).
750 */
751 if (bintime_isset(&gap_lerp)) {
752 ffdelta = cest->update_ffcount;
753 ffdelta -= fftimehands->cest.update_ffcount;
754 ffclock_convert_delta(ffdelta, cest->period, &bt);
755 polling = bt.sec;
756 bt.sec = 0;
757 bt.frac = 5000000 * (uint64_t)18446744073LL;
758 bintime_mul(&bt, polling);
759 if (bintime_cmp(&gap_lerp, &bt, >))
760 gap_lerp = bt;
761
762 /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */
763 frac = 0;
764 if (gap_lerp.sec > 0) {
765 frac -= 1;
766 frac /= ffdelta / gap_lerp.sec;
767 }
768 frac += gap_lerp.frac / ffdelta;
769
770 if (forward_jump)
771 ffth->period_lerp += frac;
772 else
773 ffth->period_lerp -= frac;
774 }
775
776 ffclock_updated = 0;
777 }
778 if (++ogen == 0)
779 ogen = 1;
780 ffth->gen = ogen;
781 fftimehands = ffth;
782}
783
784/*
785 * Adjust the fftimehands when the timecounter is changed. Stating the obvious,
786 * the old and new hardware counter cannot be read simultaneously. tc_windup()
787 * does read the two counters 'back to back', but a few cycles are effectively
788 * lost, and not accumulated in tick_ffcount. This is a fairly radical
789 * operation for a feed-forward synchronization daemon, and it is its job to not
790 * pushing irrelevant data to the kernel. Because there is no locking here,
791 * simply force to ignore pending or next update to give daemon a chance to
792 * realize the counter has changed.
793 */
794static void
795ffclock_change_tc(struct timehands *th)
796{
797 struct fftimehands *ffth;
798 struct ffclock_estimate *cest;
799 struct timecounter *tc;
800 uint8_t ogen;
801
802 tc = th->th_counter;
803 ffth = fftimehands->next;
804 ogen = ffth->gen;
805 ffth->gen = 0;
806
807 cest = &ffth->cest;
808 bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate));
809 cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1;
810 cest->errb_abs = 0;
811 cest->errb_rate = 0;
812 cest->status |= FFCLOCK_STA_UNSYNC;
813
814 ffth->tick_ffcount = fftimehands->tick_ffcount;
815 ffth->tick_time_lerp = fftimehands->tick_time_lerp;
816 ffth->tick_time = fftimehands->tick_time;
817 ffth->period_lerp = cest->period;
818
819 /* Do not lock but ignore next update from synchronization daemon. */
820 ffclock_updated--;
821
822 if (++ogen == 0)
823 ogen = 1;
824 ffth->gen = ogen;
825 fftimehands = ffth;
826}
827
828/*
829 * Retrieve feed-forward counter and time of last kernel tick.
830 */
831void
832ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags)
833{
834 struct fftimehands *ffth;
835 uint8_t gen;
836
837 /*
838 * No locking but check generation has not changed. Also need to make
839 * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
840 */
841 do {
842 ffth = fftimehands;
843 gen = ffth->gen;
844 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP)
845 *bt = ffth->tick_time_lerp;
846 else
847 *bt = ffth->tick_time;
848 *ffcount = ffth->tick_ffcount;
849 } while (gen == 0 || gen != ffth->gen);
850}
851
852/*
853 * Absolute clock conversion. Low level function to convert ffcounter to
854 * bintime. The ffcounter is converted using the current ffclock period estimate
855 * or the "interpolated period" to ensure monotonicity.
856 * NOTE: this conversion may have been deferred, and the clock updated since the
857 * hardware counter has been read.
858 */
859void
860ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags)
861{
862 struct fftimehands *ffth;
863 struct bintime bt2;
864 ffcounter ffdelta;
865 uint8_t gen;
866
867 /*
868 * No locking but check generation has not changed. Also need to make
869 * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
870 */
871 do {
872 ffth = fftimehands;
873 gen = ffth->gen;
874 if (ffcount > ffth->tick_ffcount)
875 ffdelta = ffcount - ffth->tick_ffcount;
876 else
877 ffdelta = ffth->tick_ffcount - ffcount;
878
879 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) {
880 *bt = ffth->tick_time_lerp;
881 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2);
882 } else {
883 *bt = ffth->tick_time;
884 ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2);
885 }
886
887 if (ffcount > ffth->tick_ffcount)
888 bintime_add(bt, &bt2);
889 else
890 bintime_sub(bt, &bt2);
891 } while (gen == 0 || gen != ffth->gen);
892}
893
894/*
895 * Difference clock conversion.
896 * Low level function to Convert a time interval measured in RAW counter units
897 * into bintime. The difference clock allows measuring small intervals much more
898 * reliably than the absolute clock.
899 */
900void
901ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt)
902{
903 struct fftimehands *ffth;
904 uint8_t gen;
905
906 /* No locking but check generation has not changed. */
907 do {
908 ffth = fftimehands;
909 gen = ffth->gen;
910 ffclock_convert_delta(ffdelta, ffth->cest.period, bt);
911 } while (gen == 0 || gen != ffth->gen);
912}
913
914/*
915 * Access to current ffcounter value.
916 */
917void
918ffclock_read_counter(ffcounter *ffcount)
919{
920 struct timehands *th;
921 struct fftimehands *ffth;
922 unsigned int gen, delta;
923
924 /*
925 * ffclock_windup() called from tc_windup(), safe to rely on
926 * th->th_generation only, for correct delta and ffcounter.
927 */
928 do {
929 th = timehands;
930 gen = atomic_load_acq_int(&th->th_generation);
931 ffth = fftimehands;
932 delta = tc_delta(th);
933 *ffcount = ffth->tick_ffcount;
934 atomic_thread_fence_acq();
935 } while (gen == 0 || gen != th->th_generation);
936
937 *ffcount += delta;
938}
939
940void
941binuptime(struct bintime *bt)
942{
943
944 binuptime_fromclock(bt, sysclock_active);
945}
946
947void
948nanouptime(struct timespec *tsp)
949{
950
951 nanouptime_fromclock(tsp, sysclock_active);
952}
953
954void
955microuptime(struct timeval *tvp)
956{
957
958 microuptime_fromclock(tvp, sysclock_active);
959}
960
961void
962bintime(struct bintime *bt)
963{
964
965 bintime_fromclock(bt, sysclock_active);
966}
967
968void
969nanotime(struct timespec *tsp)
970{
971
972 nanotime_fromclock(tsp, sysclock_active);
973}
974
975void
976microtime(struct timeval *tvp)
977{
978
979 microtime_fromclock(tvp, sysclock_active);
980}
981
982void
983getbinuptime(struct bintime *bt)
984{
985
986 getbinuptime_fromclock(bt, sysclock_active);
987}
988
989void
990getnanouptime(struct timespec *tsp)
991{
992
993 getnanouptime_fromclock(tsp, sysclock_active);
994}
995
996void
997getmicrouptime(struct timeval *tvp)
998{
999
1000 getmicrouptime_fromclock(tvp, sysclock_active);
1001}
1002
1003void
1004getbintime(struct bintime *bt)
1005{
1006
1007 getbintime_fromclock(bt, sysclock_active);
1008}
1009
1010void
1011getnanotime(struct timespec *tsp)
1012{
1013
1014 getnanotime_fromclock(tsp, sysclock_active);
1015}
1016
1017void
1018getmicrotime(struct timeval *tvp)
1019{
1020
1021 getmicrouptime_fromclock(tvp, sysclock_active);
1022}
1023
1024#endif /* FFCLOCK */
1025
1026/*
1027 * This is a clone of getnanotime and used for walltimestamps.
1028 * The dtrace_ prefix prevents fbt from creating probes for
1029 * it so walltimestamp can be safely used in all fbt probes.
1030 */
1031void
1032dtrace_getnanotime(struct timespec *tsp)
1033{
1034 struct timehands *th;
1035 u_int gen;
1036
1037 do {
1038 th = timehands;
1039 gen = atomic_load_acq_int(&th->th_generation);
1040 *tsp = th->th_nanotime;
1041 atomic_thread_fence_acq();
1042 } while (gen == 0 || gen != th->th_generation);
1043}
1044
1045/*
1046 * System clock currently providing time to the system. Modifiable via sysctl
1047 * when the FFCLOCK option is defined.
1048 */
1049int sysclock_active = SYSCLOCK_FBCK;
1050
1051/* Internal NTP status and error estimates. */
1052extern int time_status;
1053extern long time_esterror;
1054
1055/*
1056 * Take a snapshot of sysclock data which can be used to compare system clocks
1057 * and generate timestamps after the fact.
1058 */
1059void
1060sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast)
1061{
1062 struct fbclock_info *fbi;
1063 struct timehands *th;
1064 struct bintime bt;
1065 unsigned int delta, gen;
1066#ifdef FFCLOCK
1067 ffcounter ffcount;
1068 struct fftimehands *ffth;
1069 struct ffclock_info *ffi;
1070 struct ffclock_estimate cest;
1071
1072 ffi = &clock_snap->ff_info;
1073#endif
1074
1075 fbi = &clock_snap->fb_info;
1076 delta = 0;
1077
1078 do {
1079 th = timehands;
1080 gen = atomic_load_acq_int(&th->th_generation);
1081 fbi->th_scale = th->th_scale;
1082 fbi->tick_time = th->th_offset;
1083#ifdef FFCLOCK
1084 ffth = fftimehands;
1085 ffi->tick_time = ffth->tick_time_lerp;
1086 ffi->tick_time_lerp = ffth->tick_time_lerp;
1087 ffi->period = ffth->cest.period;
1088 ffi->period_lerp = ffth->period_lerp;
1089 clock_snap->ffcount = ffth->tick_ffcount;
1090 cest = ffth->cest;
1091#endif
1092 if (!fast)
1093 delta = tc_delta(th);
1094 atomic_thread_fence_acq();
1095 } while (gen == 0 || gen != th->th_generation);
1096
1097 clock_snap->delta = delta;
1098 clock_snap->sysclock_active = sysclock_active;
1099
1100 /* Record feedback clock status and error. */
1101 clock_snap->fb_info.status = time_status;
1102 /* XXX: Very crude estimate of feedback clock error. */
1103 bt.sec = time_esterror / 1000000;
1104 bt.frac = ((time_esterror - bt.sec) * 1000000) *
1105 (uint64_t)18446744073709ULL;
1106 clock_snap->fb_info.error = bt;
1107
1108#ifdef FFCLOCK
1109 if (!fast)
1110 clock_snap->ffcount += delta;
1111
1112 /* Record feed-forward clock leap second adjustment. */
1113 ffi->leapsec_adjustment = cest.leapsec_total;
1114 if (clock_snap->ffcount > cest.leapsec_next)
1115 ffi->leapsec_adjustment -= cest.leapsec;
1116
1117 /* Record feed-forward clock status and error. */
1118 clock_snap->ff_info.status = cest.status;
1119 ffcount = clock_snap->ffcount - cest.update_ffcount;
1120 ffclock_convert_delta(ffcount, cest.period, &bt);
1121 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */
1122 bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL);
1123 /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */
1124 bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL);
1125 clock_snap->ff_info.error = bt;
1126#endif
1127}
1128
1129/*
1130 * Convert a sysclock snapshot into a struct bintime based on the specified
1131 * clock source and flags.
1132 */
1133int
1134sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt,
1135 int whichclock, uint32_t flags)
1136{
1137 struct bintime boottimebin_x;
1138#ifdef FFCLOCK
1139 struct bintime bt2;
1140 uint64_t period;
1141#endif
1142
1143 switch (whichclock) {
1144 case SYSCLOCK_FBCK:
1145 *bt = cs->fb_info.tick_time;
1146
1147 /* If snapshot was created with !fast, delta will be >0. */
1148 if (cs->delta > 0)
1149 bintime_addx(bt, cs->fb_info.th_scale * cs->delta);
1150
1151 if ((flags & FBCLOCK_UPTIME) == 0) {
1152 getboottimebin(&boottimebin_x);
1153 bintime_add(bt, &boottimebin_x);
1154 }
1155 break;
1156#ifdef FFCLOCK
1157 case SYSCLOCK_FFWD:
1158 if (flags & FFCLOCK_LERP) {
1159 *bt = cs->ff_info.tick_time_lerp;
1160 period = cs->ff_info.period_lerp;
1161 } else {
1162 *bt = cs->ff_info.tick_time;
1163 period = cs->ff_info.period;
1164 }
1165
1166 /* If snapshot was created with !fast, delta will be >0. */
1167 if (cs->delta > 0) {
1168 ffclock_convert_delta(cs->delta, period, &bt2);
1169 bintime_add(bt, &bt2);
1170 }
1171
1172 /* Leap second adjustment. */
1173 if (flags & FFCLOCK_LEAPSEC)
1174 bt->sec -= cs->ff_info.leapsec_adjustment;
1175
1176 /* Boot time adjustment, for uptime/monotonic clocks. */
1177 if (flags & FFCLOCK_UPTIME)
1178 bintime_sub(bt, &ffclock_boottime);
1179 break;
1180#endif
1181 default:
1182 return (EINVAL);
1183 break;
1184 }
1185
1186 return (0);
1187}
1188
1189/*
1190 * Initialize a new timecounter and possibly use it.
1191 */
1192void
1193tc_init(struct timecounter *tc)
1194{
1195 u_int u;
1196 struct sysctl_oid *tc_root;
1197
1198 u = tc->tc_frequency / tc->tc_counter_mask;
1199 /* XXX: We need some margin here, 10% is a guess */
1200 u *= 11;
1201 u /= 10;
1202 if (u > hz && tc->tc_quality >= 0) {
1203 tc->tc_quality = -2000;
1204 if (bootverbose) {
1205 printf("Timecounter \"%s\" frequency %ju Hz",
1206 tc->tc_name, (uintmax_t)tc->tc_frequency);
1207 printf(" -- Insufficient hz, needs at least %u\n", u);
1208 }
1209 } else if (tc->tc_quality >= 0 || bootverbose) {
1210 printf("Timecounter \"%s\" frequency %ju Hz quality %d\n",
1211 tc->tc_name, (uintmax_t)tc->tc_frequency,
1212 tc->tc_quality);
1213 }
1214
1215 tc->tc_next = timecounters;
1216 timecounters = tc;
1217 /*
1218 * Set up sysctl tree for this counter.
1219 */
1220 tc_root = SYSCTL_ADD_NODE(NULL,
1221 SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name,
1222 CTLFLAG_RW, 0, "timecounter description");
1223 SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1224 "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0,
1225 "mask for implemented bits");
1226 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1227 "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc),
1228 sysctl_kern_timecounter_get, "IU", "current timecounter value");
1229 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1230 "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc),
1231 sysctl_kern_timecounter_freq, "QU", "timecounter frequency");
1232 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1233 "quality", CTLFLAG_RD, &(tc->tc_quality), 0,
1234 "goodness of time counter");
1235 /*
1236 * Do not automatically switch if the current tc was specifically
1237 * chosen. Never automatically use a timecounter with negative quality.
1238 * Even though we run on the dummy counter, switching here may be
1239 * worse since this timecounter may not be monotonic.
1240 */
1241 if (tc_chosen)
1242 return;
1243 if (tc->tc_quality < 0)
1244 return;
1245 if (tc->tc_quality < timecounter->tc_quality)
1246 return;
1247 if (tc->tc_quality == timecounter->tc_quality &&
1248 tc->tc_frequency < timecounter->tc_frequency)
1249 return;
1250 (void)tc->tc_get_timecount(tc);
1251 (void)tc->tc_get_timecount(tc);
1252 timecounter = tc;
1253}
1254
1255/* Report the frequency of the current timecounter. */
1256uint64_t
1257tc_getfrequency(void)
1258{
1259
1260 return (timehands->th_counter->tc_frequency);
1261}
1262
1263static struct mtx tc_setclock_mtx;
1264MTX_SYSINIT(tc_setclock_init, &tc_setclock_mtx, "tcsetc", MTX_SPIN);
1265
1266/*
1267 * Step our concept of UTC. This is done by modifying our estimate of
1268 * when we booted.
1269 */
1270void
1271tc_setclock(struct timespec *ts)
1272{
1273 struct timespec tbef, taft;
1274 struct bintime bt, bt2;
1275
1276 timespec2bintime(ts, &bt);
1277 nanotime(&tbef);
1278 mtx_lock_spin(&tc_setclock_mtx);
1279 cpu_tick_calibrate(1);
1280 binuptime(&bt2);
1281 bintime_sub(&bt, &bt2);
1282
1283 /* XXX fiddle all the little crinkly bits around the fiords... */
1284 tc_windup(&bt);
1285 mtx_unlock_spin(&tc_setclock_mtx);
1286 getboottimebin(&boottimebin);
1287 bintime2timeval(&boottimebin, &boottime);
1288 if (timestepwarnings) {
1289 nanotime(&taft);
1290 log(LOG_INFO,
1291 "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n",
1292 (intmax_t)tbef.tv_sec, tbef.tv_nsec,
1293 (intmax_t)taft.tv_sec, taft.tv_nsec,
1294 (intmax_t)ts->tv_sec, ts->tv_nsec);
1295 }
1296}
1297
1298/*
1299 * Initialize the next struct timehands in the ring and make
1300 * it the active timehands. Along the way we might switch to a different
1301 * timecounter and/or do seconds processing in NTP. Slightly magic.
1302 */
1303static void
1304tc_windup(struct bintime *new_boottimebin)
1305{
1306 struct bintime bt;
1307 struct timehands *th, *tho;
1308 uint64_t scale;
1309 u_int delta, ncount, ogen;
1310 int i;
1311 time_t t;
1312
1313 /*
1314 * Make the next timehands a copy of the current one, but do
1315 * not overwrite the generation or next pointer. While we
1316 * update the contents, the generation must be zero. We need
1317 * to ensure that the zero generation is visible before the
1318 * data updates become visible, which requires release fence.
1319 * For similar reasons, re-reading of the generation after the
1320 * data is read should use acquire fence.
1321 */
1322 tho = timehands;
1323 th = tho->th_next;
1324 ogen = th->th_generation;
1325 th->th_generation = 0;
1326 atomic_thread_fence_rel();
1327 bcopy(tho, th, offsetof(struct timehands, th_generation));
1328 if (new_boottimebin != NULL)
1329 th->th_boottime = *new_boottimebin;
1330
1331 /*
1332 * Capture a timecounter delta on the current timecounter and if
1333 * changing timecounters, a counter value from the new timecounter.
1334 * Update the offset fields accordingly.
1335 */
1336 delta = tc_delta(th);
1337 if (th->th_counter != timecounter)
1338 ncount = timecounter->tc_get_timecount(timecounter);
1339 else
1340 ncount = 0;
1341#ifdef FFCLOCK
1342 ffclock_windup(delta);
1343#endif
1344 th->th_offset_count += delta;
1345 th->th_offset_count &= th->th_counter->tc_counter_mask;
1346 while (delta > th->th_counter->tc_frequency) {
1347 /* Eat complete unadjusted seconds. */
1348 delta -= th->th_counter->tc_frequency;
1349 th->th_offset.sec++;
1350 }
1351 if ((delta > th->th_counter->tc_frequency / 2) &&
1352 (th->th_scale * delta < ((uint64_t)1 << 63))) {
1353 /* The product th_scale * delta just barely overflows. */
1354 th->th_offset.sec++;
1355 }
1356 bintime_addx(&th->th_offset, th->th_scale * delta);
1357
1358 /*
1359 * Hardware latching timecounters may not generate interrupts on
1360 * PPS events, so instead we poll them. There is a finite risk that
1361 * the hardware might capture a count which is later than the one we
1362 * got above, and therefore possibly in the next NTP second which might
1363 * have a different rate than the current NTP second. It doesn't
1364 * matter in practice.
1365 */
1366 if (tho->th_counter->tc_poll_pps)
1367 tho->th_counter->tc_poll_pps(tho->th_counter);
1368
1369 /*
1370 * Deal with NTP second processing. The for loop normally
1371 * iterates at most once, but in extreme situations it might
1372 * keep NTP sane if timeouts are not run for several seconds.
1373 * At boot, the time step can be large when the TOD hardware
1374 * has been read, so on really large steps, we call
1375 * ntp_update_second only twice. We need to call it twice in
1376 * case we missed a leap second.
1377 */
1378 bt = th->th_offset;
1379 bintime_add(&bt, &th->th_boottime);
1380 i = bt.sec - tho->th_microtime.tv_sec;
1381 if (i > LARGE_STEP)
1382 i = 2;
1383 for (; i > 0; i--) {
1384 t = bt.sec;
1385 ntp_update_second(&th->th_adjustment, &bt.sec);
1386 if (bt.sec != t)
1387 th->th_boottime.sec += bt.sec - t;
1388 }
1389 th->th_bintime = th->th_offset;
1390 bintime_add(&th->th_bintime, &th->th_boottime);
1391 /* Update the UTC timestamps used by the get*() functions. */
1392 /* XXX shouldn't do this here. Should force non-`get' versions. */
1393 bintime2timeval(&bt, &th->th_microtime);
1394 bintime2timespec(&bt, &th->th_nanotime);
1395
1396 /* Now is a good time to change timecounters. */
1397 if (th->th_counter != timecounter) {
1398#ifndef __arm__
1399 if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0)
1400 cpu_disable_c2_sleep++;
1401 if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0)
1402 cpu_disable_c2_sleep--;
1403#endif
1404 th->th_counter = timecounter;
1405 th->th_offset_count = ncount;
1406 tc_min_ticktock_freq = max(1, timecounter->tc_frequency /
1407 (((uint64_t)timecounter->tc_counter_mask + 1) / 3));
1408#ifdef FFCLOCK
1409 ffclock_change_tc(th);
1410#endif
1411 }
1412
1413 /*-
1414 * Recalculate the scaling factor. We want the number of 1/2^64
1415 * fractions of a second per period of the hardware counter, taking
1416 * into account the th_adjustment factor which the NTP PLL/adjtime(2)
1417 * processing provides us with.
1418 *
1419 * The th_adjustment is nanoseconds per second with 32 bit binary
1420 * fraction and we want 64 bit binary fraction of second:
1421 *
1422 * x = a * 2^32 / 10^9 = a * 4.294967296
1423 *
1424 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int
1425 * we can only multiply by about 850 without overflowing, that
1426 * leaves no suitably precise fractions for multiply before divide.
1427 *
1428 * Divide before multiply with a fraction of 2199/512 results in a
1429 * systematic undercompensation of 10PPM of th_adjustment. On a
1430 * 5000PPM adjustment this is a 0.05PPM error. This is acceptable.
1431 *
1432 * We happily sacrifice the lowest of the 64 bits of our result
1433 * to the goddess of code clarity.
1434 *
1435 */
1436 scale = (uint64_t)1 << 63;
1437 scale += (th->th_adjustment / 1024) * 2199;
1438 scale /= th->th_counter->tc_frequency;
1439 th->th_scale = scale * 2;
1440
1441 /*
1442 * Now that the struct timehands is again consistent, set the new
1443 * generation number, making sure to not make it zero.
1444 */
1445 if (++ogen == 0)
1446 ogen = 1;
1447 atomic_store_rel_int(&th->th_generation, ogen);
1448
1449 /* Go live with the new struct timehands. */
1450#ifdef FFCLOCK
1451 switch (sysclock_active) {
1452 case SYSCLOCK_FBCK:
1453#endif
1454 time_second = th->th_microtime.tv_sec;
1455 time_uptime = th->th_offset.sec;
1456#ifdef FFCLOCK
1457 break;
1458 case SYSCLOCK_FFWD:
1459 time_second = fftimehands->tick_time_lerp.sec;
1460 time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec;
1461 break;
1462 }
1463#endif
1464
1465 timehands = th;
1466 timekeep_push_vdso();
1467}
1468
1469/* Report or change the active timecounter hardware. */
1470static int
1471sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS)
1472{
1473 char newname[32];
1474 struct timecounter *newtc, *tc;
1475 int error;
1476
1477 tc = timecounter;
1478 strlcpy(newname, tc->tc_name, sizeof(newname));
1479
1480 error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req);
1481 if (error != 0 || req->newptr == NULL)
1482 return (error);
1483 /* Record that the tc in use now was specifically chosen. */
1484 tc_chosen = 1;
1485 if (strcmp(newname, tc->tc_name) == 0)
1486 return (0);
1487 for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) {
1488 if (strcmp(newname, newtc->tc_name) != 0)
1489 continue;
1490
1491 /* Warm up new timecounter. */
1492 (void)newtc->tc_get_timecount(newtc);
1493 (void)newtc->tc_get_timecount(newtc);
1494
1495 timecounter = newtc;
1496
1497 /*
1498 * The vdso timehands update is deferred until the next
1499 * 'tc_windup()'.
1500 *
1501 * This is prudent given that 'timekeep_push_vdso()' does not
1502 * use any locking and that it can be called in hard interrupt
1503 * context via 'tc_windup()'.
1504 */
1505 return (0);
1506 }
1507 return (EINVAL);
1508}
1509
1510SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW,
1511 0, 0, sysctl_kern_timecounter_hardware, "A",
1512 "Timecounter hardware selected");
1513
1514
1515/* Report the available timecounter hardware. */
1516static int
1517sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS)
1518{
1519 struct sbuf sb;
1520 struct timecounter *tc;
1521 int error;
1522
1523 sbuf_new_for_sysctl(&sb, NULL, 0, req);
1524 for (tc = timecounters; tc != NULL; tc = tc->tc_next) {
1525 if (tc != timecounters)
1526 sbuf_putc(&sb, ' ');
1527 sbuf_printf(&sb, "%s(%d)", tc->tc_name, tc->tc_quality);
1528 }
1529 error = sbuf_finish(&sb);
1530 sbuf_delete(&sb);
1531 return (error);
1532}
1533
1534SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD,
1535 0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected");
1536
1537/*
1538 * RFC 2783 PPS-API implementation.
1539 */
1540
1541/*
1542 * Return true if the driver is aware of the abi version extensions in the
1543 * pps_state structure, and it supports at least the given abi version number.
1544 */
1545static inline int
1546abi_aware(struct pps_state *pps, int vers)
1547{
1548
1549 return ((pps->kcmode & KCMODE_ABIFLAG) && pps->driver_abi >= vers);
1550}
1551
1552static int
1553pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps)
1554{
1555 int err, timo;
1556 pps_seq_t aseq, cseq;
1557 struct timeval tv;
1558
1559 if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC)
1560 return (EINVAL);
1561
1562 /*
1563 * If no timeout is requested, immediately return whatever values were
1564 * most recently captured. If timeout seconds is -1, that's a request
1565 * to block without a timeout. WITNESS won't let us sleep forever
1566 * without a lock (we really don't need a lock), so just repeatedly
1567 * sleep a long time.
1568 */
1569 if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) {
1570 if (fapi->timeout.tv_sec == -1)
1571 timo = 0x7fffffff;
1572 else {
1573 tv.tv_sec = fapi->timeout.tv_sec;
1574 tv.tv_usec = fapi->timeout.tv_nsec / 1000;
1575 timo = tvtohz(&tv);
1576 }
1577 aseq = pps->ppsinfo.assert_sequence;
1578 cseq = pps->ppsinfo.clear_sequence;
1579 while (aseq == pps->ppsinfo.assert_sequence &&
1580 cseq == pps->ppsinfo.clear_sequence) {
1581 if (abi_aware(pps, 1) && pps->driver_mtx != NULL) {
1582 if (pps->flags & PPSFLAG_MTX_SPIN) {
1583 err = msleep_spin(pps, pps->driver_mtx,
1584 "ppsfch", timo);
1585 } else {
1586 err = msleep(pps, pps->driver_mtx, PCATCH,
1587 "ppsfch", timo);
1588 }
1589 } else {
1590 err = tsleep(pps, PCATCH, "ppsfch", timo);
1591 }
1592 if (err == EWOULDBLOCK) {
1593 if (fapi->timeout.tv_sec == -1) {
1594 continue;
1595 } else {
1596 return (ETIMEDOUT);
1597 }
1598 } else if (err != 0) {
1599 return (err);
1600 }
1601 }
1602 }
1603
1604 pps->ppsinfo.current_mode = pps->ppsparam.mode;
1605 fapi->pps_info_buf = pps->ppsinfo;
1606
1607 return (0);
1608}
1609
1610int
1611pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps)
1612{
1613 pps_params_t *app;
1614 struct pps_fetch_args *fapi;
1615#ifdef FFCLOCK
1616 struct pps_fetch_ffc_args *fapi_ffc;
1617#endif
1618#ifdef PPS_SYNC
1619 struct pps_kcbind_args *kapi;
1620#endif
1621
1622 KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl"));
1623 switch (cmd) {
1624 case PPS_IOC_CREATE:
1625 return (0);
1626 case PPS_IOC_DESTROY:
1627 return (0);
1628 case PPS_IOC_SETPARAMS:
1629 app = (pps_params_t *)data;
1630 if (app->mode & ~pps->ppscap)
1631 return (EINVAL);
1632#ifdef FFCLOCK
1633 /* Ensure only a single clock is selected for ffc timestamp. */
1634 if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK)
1635 return (EINVAL);
1636#endif
1637 pps->ppsparam = *app;
1638 return (0);
1639 case PPS_IOC_GETPARAMS:
1640 app = (pps_params_t *)data;
1641 *app = pps->ppsparam;
1642 app->api_version = PPS_API_VERS_1;
1643 return (0);
1644 case PPS_IOC_GETCAP:
1645 *(int*)data = pps->ppscap;
1646 return (0);
1647 case PPS_IOC_FETCH:
1648 fapi = (struct pps_fetch_args *)data;
1649 return (pps_fetch(fapi, pps));
1650#ifdef FFCLOCK
1651 case PPS_IOC_FETCH_FFCOUNTER:
1652 fapi_ffc = (struct pps_fetch_ffc_args *)data;
1653 if (fapi_ffc->tsformat && fapi_ffc->tsformat !=
1654 PPS_TSFMT_TSPEC)
1655 return (EINVAL);
1656 if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec)
1657 return (EOPNOTSUPP);
1658 pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode;
1659 fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc;
1660 /* Overwrite timestamps if feedback clock selected. */
1661 switch (pps->ppsparam.mode & PPS_TSCLK_MASK) {
1662 case PPS_TSCLK_FBCK:
1663 fapi_ffc->pps_info_buf_ffc.assert_timestamp =
1664 pps->ppsinfo.assert_timestamp;
1665 fapi_ffc->pps_info_buf_ffc.clear_timestamp =
1666 pps->ppsinfo.clear_timestamp;
1667 break;
1668 case PPS_TSCLK_FFWD:
1669 break;
1670 default:
1671 break;
1672 }
1673 return (0);
1674#endif /* FFCLOCK */
1675 case PPS_IOC_KCBIND:
1676#ifdef PPS_SYNC
1677 kapi = (struct pps_kcbind_args *)data;
1678 /* XXX Only root should be able to do this */
1679 if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC)
1680 return (EINVAL);
1681 if (kapi->kernel_consumer != PPS_KC_HARDPPS)
1682 return (EINVAL);
1683 if (kapi->edge & ~pps->ppscap)
1684 return (EINVAL);
1685 pps->kcmode = (kapi->edge & KCMODE_EDGEMASK) |
1686 (pps->kcmode & KCMODE_ABIFLAG);
1687 return (0);
1688#else
1689 return (EOPNOTSUPP);
1690#endif
1691 default:
1692 return (ENOIOCTL);
1693 }
1694}
1695
1696void
1697pps_init(struct pps_state *pps)
1698{
1699 pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT;
1700 if (pps->ppscap & PPS_CAPTUREASSERT)
1701 pps->ppscap |= PPS_OFFSETASSERT;
1702 if (pps->ppscap & PPS_CAPTURECLEAR)
1703 pps->ppscap |= PPS_OFFSETCLEAR;
1704#ifdef FFCLOCK
1705 pps->ppscap |= PPS_TSCLK_MASK;
1706#endif
1707 pps->kcmode &= ~KCMODE_ABIFLAG;
1708}
1709
1710void
1711pps_init_abi(struct pps_state *pps)
1712{
1713
1714 pps_init(pps);
1715 if (pps->driver_abi > 0) {
1716 pps->kcmode |= KCMODE_ABIFLAG;
1717 pps->kernel_abi = PPS_ABI_VERSION;
1718 }
1719}
1720
1721void
1722pps_capture(struct pps_state *pps)
1723{
1724 struct timehands *th;
1725
1726 KASSERT(pps != NULL, ("NULL pps pointer in pps_capture"));
1727 th = timehands;
1728 pps->capgen = atomic_load_acq_int(&th->th_generation);
1729 pps->capth = th;
1730#ifdef FFCLOCK
1731 pps->capffth = fftimehands;
1732#endif
1733 pps->capcount = th->th_counter->tc_get_timecount(th->th_counter);
1734 atomic_thread_fence_acq();
1735 if (pps->capgen != th->th_generation)
1736 pps->capgen = 0;
1737}
1738
1739void
1740pps_event(struct pps_state *pps, int event)
1741{
1742 struct bintime bt;
1743 struct timespec ts, *tsp, *osp;
1744 u_int tcount, *pcount;
1745 int foff;
1746 pps_seq_t *pseq;
1747#ifdef FFCLOCK
1748 struct timespec *tsp_ffc;
1749 pps_seq_t *pseq_ffc;
1750 ffcounter *ffcount;
1751#endif
1752#ifdef PPS_SYNC
1753 int fhard;
1754#endif
1755
1756 KASSERT(pps != NULL, ("NULL pps pointer in pps_event"));
1757 /* Nothing to do if not currently set to capture this event type. */
1758 if ((event & pps->ppsparam.mode) == 0)
1759 return;
1760 /* If the timecounter was wound up underneath us, bail out. */
1761 if (pps->capgen == 0 || pps->capgen !=
1762 atomic_load_acq_int(&pps->capth->th_generation))
1763 return;
1764
1765 /* Things would be easier with arrays. */
1766 if (event == PPS_CAPTUREASSERT) {
1767 tsp = &pps->ppsinfo.assert_timestamp;
1768 osp = &pps->ppsparam.assert_offset;
1769 foff = pps->ppsparam.mode & PPS_OFFSETASSERT;
1770#ifdef PPS_SYNC
1771 fhard = pps->kcmode & PPS_CAPTUREASSERT;
1772#endif
1773 pcount = &pps->ppscount[0];
1774 pseq = &pps->ppsinfo.assert_sequence;
1775#ifdef FFCLOCK
1776 ffcount = &pps->ppsinfo_ffc.assert_ffcount;
1777 tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp;
1778 pseq_ffc = &pps->ppsinfo_ffc.assert_sequence;
1779#endif
1780 } else {
1781 tsp = &pps->ppsinfo.clear_timestamp;
1782 osp = &pps->ppsparam.clear_offset;
1783 foff = pps->ppsparam.mode & PPS_OFFSETCLEAR;
1784#ifdef PPS_SYNC
1785 fhard = pps->kcmode & PPS_CAPTURECLEAR;
1786#endif
1787 pcount = &pps->ppscount[1];
1788 pseq = &pps->ppsinfo.clear_sequence;
1789#ifdef FFCLOCK
1790 ffcount = &pps->ppsinfo_ffc.clear_ffcount;
1791 tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp;
1792 pseq_ffc = &pps->ppsinfo_ffc.clear_sequence;
1793#endif
1794 }
1795
1796 /*
1797 * If the timecounter changed, we cannot compare the count values, so
1798 * we have to drop the rest of the PPS-stuff until the next event.
1799 */
1800 if (pps->ppstc != pps->capth->th_counter) {
1801 pps->ppstc = pps->capth->th_counter;
1802 *pcount = pps->capcount;
1803 pps->ppscount[2] = pps->capcount;
1804 return;
1805 }
1806
1807 /* Convert the count to a timespec. */
1808 tcount = pps->capcount - pps->capth->th_offset_count;
1809 tcount &= pps->capth->th_counter->tc_counter_mask;
1810 bt = pps->capth->th_bintime;
1811 bintime_addx(&bt, pps->capth->th_scale * tcount);
1812 bintime2timespec(&bt, &ts);
1813
1814 /* If the timecounter was wound up underneath us, bail out. */
1815 atomic_thread_fence_acq();
1816 if (pps->capgen != pps->capth->th_generation)
1817 return;
1818
1819 *pcount = pps->capcount;
1820 (*pseq)++;
1821 *tsp = ts;
1822
1823 if (foff) {
1824 timespecadd(tsp, osp);
1825 if (tsp->tv_nsec < 0) {
1826 tsp->tv_nsec += 1000000000;
1827 tsp->tv_sec -= 1;
1828 }
1829 }
1830
1831#ifdef FFCLOCK
1832 *ffcount = pps->capffth->tick_ffcount + tcount;
1833 bt = pps->capffth->tick_time;
1834 ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt);
1835 bintime_add(&bt, &pps->capffth->tick_time);
1836 bintime2timespec(&bt, &ts);
1837 (*pseq_ffc)++;
1838 *tsp_ffc = ts;
1839#endif
1840
1841#ifdef PPS_SYNC
1842 if (fhard) {
1843 uint64_t scale;
1844
1845 /*
1846 * Feed the NTP PLL/FLL.
1847 * The FLL wants to know how many (hardware) nanoseconds
1848 * elapsed since the previous event.
1849 */
1850 tcount = pps->capcount - pps->ppscount[2];
1851 pps->ppscount[2] = pps->capcount;
1852 tcount &= pps->capth->th_counter->tc_counter_mask;
1853 scale = (uint64_t)1 << 63;
1854 scale /= pps->capth->th_counter->tc_frequency;
1855 scale *= 2;
1856 bt.sec = 0;
1857 bt.frac = 0;
1858 bintime_addx(&bt, scale * tcount);
1859 bintime2timespec(&bt, &ts);
1860 hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec);
1861 }
1862#endif
1863
1864 /* Wakeup anyone sleeping in pps_fetch(). */
1865 wakeup(pps);
1866}
1867
1868/*
1869 * Timecounters need to be updated every so often to prevent the hardware
1870 * counter from overflowing. Updating also recalculates the cached values
1871 * used by the get*() family of functions, so their precision depends on
1872 * the update frequency.
1873 */
1874
1875static int tc_tick;
1876SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0,
1877 "Approximate number of hardclock ticks in a millisecond");
1878
1879void
1880tc_ticktock(int cnt)
1881{
1882 static int count;
1883
1884 if (mtx_trylock_spin(&tc_setclock_mtx)) {
1885 count += cnt;
1886 if (count >= tc_tick) {
1887 count = 0;
1888 tc_windup(NULL);
1889 }
1890 mtx_unlock_spin(&tc_setclock_mtx);
1891 }
1892}
1893
1894static void __inline
1895tc_adjprecision(void)
1896{
1897 int t;
1898
1899 if (tc_timepercentage > 0) {
1900 t = (99 + tc_timepercentage) / tc_timepercentage;
1901 tc_precexp = fls(t + (t >> 1)) - 1;
1902 FREQ2BT(hz / tc_tick, &bt_timethreshold);
1903 FREQ2BT(hz, &bt_tickthreshold);
1904 bintime_shift(&bt_timethreshold, tc_precexp);
1905 bintime_shift(&bt_tickthreshold, tc_precexp);
1906 } else {
1907 tc_precexp = 31;
1908 bt_timethreshold.sec = INT_MAX;
1909 bt_timethreshold.frac = ~(uint64_t)0;
1910 bt_tickthreshold = bt_timethreshold;
1911 }
1912 sbt_timethreshold = bttosbt(bt_timethreshold);
1913 sbt_tickthreshold = bttosbt(bt_tickthreshold);
1914}
1915
1916static int
1917sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS)
1918{
1919 int error, val;
1920
1921 val = tc_timepercentage;
1922 error = sysctl_handle_int(oidp, &val, 0, req);
1923 if (error != 0 || req->newptr == NULL)
1924 return (error);
1925 tc_timepercentage = val;
1926 if (cold)
1927 goto done;
1928 tc_adjprecision();
1929done:
1930 return (0);
1931}
1932
1933static void
1934inittimecounter(void *dummy)
1935{
1936 u_int p;
1937 int tick_rate;
1938
1939 /*
1940 * Set the initial timeout to
1941 * max(1, <approx. number of hardclock ticks in a millisecond>).
1942 * People should probably not use the sysctl to set the timeout
1943 * to smaller than its initial value, since that value is the
1944 * smallest reasonable one. If they want better timestamps they
1945 * should use the non-"get"* functions.
1946 */
1947 if (hz > 1000)
1948 tc_tick = (hz + 500) / 1000;
1949 else
1950 tc_tick = 1;
1951 tc_adjprecision();
1952 FREQ2BT(hz, &tick_bt);
1953 tick_sbt = bttosbt(tick_bt);
1954 tick_rate = hz / tc_tick;
1955 FREQ2BT(tick_rate, &tc_tick_bt);
1956 tc_tick_sbt = bttosbt(tc_tick_bt);
1957 p = (tc_tick * 1000000) / hz;
1958 printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000);
1959
1960#ifdef FFCLOCK
1961 ffclock_init();
1962#endif
1963 /* warm up new timecounter (again) and get rolling. */
1964 (void)timecounter->tc_get_timecount(timecounter);
1965 (void)timecounter->tc_get_timecount(timecounter);
1966 mtx_lock_spin(&tc_setclock_mtx);
1967 tc_windup(NULL);
1968 mtx_unlock_spin(&tc_setclock_mtx);
1969}
1970
1971SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL);
1972
1973/* Cpu tick handling -------------------------------------------------*/
1974
1975static int cpu_tick_variable;
1976static uint64_t cpu_tick_frequency;
1977
1978static DPCPU_DEFINE(uint64_t, tc_cpu_ticks_base);
1979static DPCPU_DEFINE(unsigned, tc_cpu_ticks_last);
1980
1981static uint64_t
1982tc_cpu_ticks(void)
1983{
1984 struct timecounter *tc;
1985 uint64_t res, *base;
1986 unsigned u, *last;
1987
1988 critical_enter();
1989 base = DPCPU_PTR(tc_cpu_ticks_base);
1990 last = DPCPU_PTR(tc_cpu_ticks_last);
1991 tc = timehands->th_counter;
1992 u = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
1993 if (u < *last)
1994 *base += (uint64_t)tc->tc_counter_mask + 1;
1995 *last = u;
1996 res = u + *base;
1997 critical_exit();
1998 return (res);
1999}
2000
2001void
2002cpu_tick_calibration(void)
2003{
2004 static time_t last_calib;
2005
2006 if (time_uptime != last_calib && !(time_uptime & 0xf)) {
2007 cpu_tick_calibrate(0);
2008 last_calib = time_uptime;
2009 }
2010}
2011
2012/*
2013 * This function gets called every 16 seconds on only one designated
2014 * CPU in the system from hardclock() via cpu_tick_calibration()().
2015 *
2016 * Whenever the real time clock is stepped we get called with reset=1
2017 * to make sure we handle suspend/resume and similar events correctly.
2018 */
2019
2020static void
2021cpu_tick_calibrate(int reset)
2022{
2023 static uint64_t c_last;
2024 uint64_t c_this, c_delta;
2025 static struct bintime t_last;
2026 struct bintime t_this, t_delta;
2027 uint32_t divi;
2028
2029 if (reset) {
2030 /* The clock was stepped, abort & reset */
2031 t_last.sec = 0;
2032 return;
2033 }
2034
2035 /* we don't calibrate fixed rate cputicks */
2036 if (!cpu_tick_variable)
2037 return;
2038
2039 getbinuptime(&t_this);
2040 c_this = cpu_ticks();
2041 if (t_last.sec != 0) {
2042 c_delta = c_this - c_last;
2043 t_delta = t_this;
2044 bintime_sub(&t_delta, &t_last);
2045 /*
2046 * Headroom:
2047 * 2^(64-20) / 16[s] =
2048 * 2^(44) / 16[s] =
2049 * 17.592.186.044.416 / 16 =
2050 * 1.099.511.627.776 [Hz]
2051 */
2052 divi = t_delta.sec << 20;
2053 divi |= t_delta.frac >> (64 - 20);
2054 c_delta <<= 20;
2055 c_delta /= divi;
2056 if (c_delta > cpu_tick_frequency) {
2057 if (0 && bootverbose)
2058 printf("cpu_tick increased to %ju Hz\n",
2059 c_delta);
2060 cpu_tick_frequency = c_delta;
2061 }
2062 }
2063 c_last = c_this;
2064 t_last = t_this;
2065}
2066
2067void
2068set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var)
2069{
2070
2071 if (func == NULL) {
2072 cpu_ticks = tc_cpu_ticks;
2073 } else {
2074 cpu_tick_frequency = freq;
2075 cpu_tick_variable = var;
2076 cpu_ticks = func;
2077 }
2078}
2079
2080uint64_t
2081cpu_tickrate(void)
2082{
2083
2084 if (cpu_ticks == tc_cpu_ticks)
2085 return (tc_getfrequency());
2086 return (cpu_tick_frequency);
2087}
2088
2089/*
2090 * We need to be slightly careful converting cputicks to microseconds.
2091 * There is plenty of margin in 64 bits of microseconds (half a million
2092 * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply
2093 * before divide conversion (to retain precision) we find that the
2094 * margin shrinks to 1.5 hours (one millionth of 146y).
2095 * With a three prong approach we never lose significant bits, no
2096 * matter what the cputick rate and length of timeinterval is.
2097 */
2098
2099uint64_t
2100cputick2usec(uint64_t tick)
2101{
2102
2103 if (tick > 18446744073709551LL) /* floor(2^64 / 1000) */
2104 return (tick / (cpu_tickrate() / 1000000LL));
2105 else if (tick > 18446744073709LL) /* floor(2^64 / 1000000) */
2106 return ((tick * 1000LL) / (cpu_tickrate() / 1000LL));
2107 else
2108 return ((tick * 1000000LL) / cpu_tickrate());
2109}
2110
2111cpu_tick_f *cpu_ticks = tc_cpu_ticks;
2112
2113static int vdso_th_enable = 1;
2114static int
2115sysctl_fast_gettime(SYSCTL_HANDLER_ARGS)
2116{
2117 int old_vdso_th_enable, error;
2118
2119 old_vdso_th_enable = vdso_th_enable;
2120 error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req);
2121 if (error != 0)
2122 return (error);
2123 vdso_th_enable = old_vdso_th_enable;
2124 return (0);
2125}
2126SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime,
2127 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
2128 NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day");
2129
2130uint32_t
2131tc_fill_vdso_timehands(struct vdso_timehands *vdso_th)
2132{
2133 struct timehands *th;
2134 uint32_t enabled;
2135
2136 th = timehands;
2137 vdso_th->th_algo = VDSO_TH_ALGO_1;
2138 vdso_th->th_scale = th->th_scale;
2139 vdso_th->th_offset_count = th->th_offset_count;
2140 vdso_th->th_counter_mask = th->th_counter->tc_counter_mask;
2141 vdso_th->th_offset = th->th_offset;
2142 vdso_th->th_boottime = th->th_boottime;
2143 enabled = cpu_fill_vdso_timehands(vdso_th, th->th_counter);
2144 if (!vdso_th_enable)
2145 enabled = 0;
2146 return (enabled);
2147}
2148
2149#ifdef COMPAT_FREEBSD32
2150uint32_t
2151tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
2152{
2153 struct timehands *th;
2154 uint32_t enabled;
2155
2156 th = timehands;
2157 vdso_th32->th_algo = VDSO_TH_ALGO_1;
2158 *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale;
2159 vdso_th32->th_offset_count = th->th_offset_count;
2160 vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask;
2161 vdso_th32->th_offset.sec = th->th_offset.sec;
2162 *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac;
2163 vdso_th32->th_boottime.sec = th->th_boottime.sec;
2164 *(uint64_t *)&vdso_th32->th_boottime.frac[0] = th->th_boottime.frac;
2165 enabled = cpu_fill_vdso_timehands32(vdso_th32, th->th_counter);
2166 if (!vdso_th_enable)
2167 enabled = 0;
2168 return (enabled);
2169}
2170#endif