1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 The University of Melbourne 5 * All rights reserved. 6 * 7 * This software was developed by Julien Ridoux at the University of Melbourne 8 * under sponsorship from the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33#include "opt_ffclock.h" 34 35#include <sys/param.h> 36#include <sys/bus.h> 37#include <sys/kernel.h> 38#include <sys/lock.h> 39#include <sys/module.h> 40#include <sys/mutex.h> 41#include <sys/priv.h> 42#include <sys/proc.h> 43#include <sys/sbuf.h> 44#include <sys/sysproto.h> 45#include <sys/sysctl.h> 46#include <sys/systm.h> 47#include <sys/timeffc.h> 48 49#ifdef FFCLOCK 50 51FEATURE(ffclock, "Feed-forward clock support"); 52 53extern struct ffclock_estimate ffclock_estimate; 54extern struct bintime ffclock_boottime; 55extern int8_t ffclock_updated; 56extern struct mtx ffclock_mtx; 57 58/* 59 * Feed-forward clock absolute time. This should be the preferred way to read 60 * the feed-forward clock for "wall-clock" type time. The flags allow to compose 61 * various flavours of absolute time (e.g. with or without leap seconds taken 62 * into account). If valid pointers are provided, the ffcounter value and an 63 * upper bound on clock error associated with the bintime are provided. 64 * NOTE: use ffclock_convert_abs() to differ the conversion of a ffcounter value 65 * read earlier. 66 */ 67void 68ffclock_abstime(ffcounter *ffcount, struct bintime *bt, 69 struct bintime *error_bound, uint32_t flags) 70{ 71 struct ffclock_estimate cest; 72 ffcounter ffc; 73 ffcounter update_ffcount; 74 ffcounter ffdelta_error; 75 76 /* Get counter and corresponding time. */ 77 if ((flags & FFCLOCK_FAST) == FFCLOCK_FAST) 78 ffclock_last_tick(&ffc, bt, flags); 79 else { 80 ffclock_read_counter(&ffc); 81 ffclock_convert_abs(ffc, bt, flags); 82 } 83 84 /* Current ffclock estimate, use update_ffcount as generation number. */ 85 do { 86 update_ffcount = ffclock_estimate.update_ffcount; 87 bcopy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate)); 88 } while (update_ffcount != ffclock_estimate.update_ffcount); 89 90 /* 91 * Leap second adjustment. Total as seen by synchronisation algorithm 92 * since it started. cest.leapsec_next is the ffcounter prediction of 93 * when the next leapsecond occurs. 94 */ 95 if ((flags & FFCLOCK_LEAPSEC) == FFCLOCK_LEAPSEC) { 96 bt->sec -= cest.leapsec_total; 97 if (ffc > cest.leapsec_next) 98 bt->sec -= cest.leapsec; 99 } 100 101 /* Boot time adjustment, for uptime/monotonic clocks. */ 102 if ((flags & FFCLOCK_UPTIME) == FFCLOCK_UPTIME) { 103 bintime_sub(bt, &ffclock_boottime); 104 } 105 106 /* Compute error bound if a valid pointer has been passed. */ 107 if (error_bound) { 108 ffdelta_error = ffc - cest.update_ffcount; 109 ffclock_convert_diff(ffdelta_error, error_bound); 110 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */ 111 bintime_mul(error_bound, cest.errb_rate * 112 (uint64_t)18446744073709LL); 113 /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns] */ 114 bintime_addx(error_bound, cest.errb_abs * 115 (uint64_t)18446744073LL); 116 } 117 118 if (ffcount) 119 *ffcount = ffc; 120} 121 122/* 123 * Feed-forward difference clock. This should be the preferred way to convert a 124 * time interval in ffcounter values into a time interval in seconds. If a valid 125 * pointer is passed, an upper bound on the error in computing the time interval 126 * in seconds is provided. 127 */ 128void 129ffclock_difftime(ffcounter ffdelta, struct bintime *bt, 130 struct bintime *error_bound) 131{ 132 ffcounter update_ffcount; 133 uint32_t err_rate; 134 135 ffclock_convert_diff(ffdelta, bt); 136 137 if (error_bound) { 138 do { 139 update_ffcount = ffclock_estimate.update_ffcount; 140 err_rate = ffclock_estimate.errb_rate; 141 } while (update_ffcount != ffclock_estimate.update_ffcount); 142 143 ffclock_convert_diff(ffdelta, error_bound); 144 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */ 145 bintime_mul(error_bound, err_rate * (uint64_t)18446744073709LL); 146 } 147} 148 149/* 150 * Create a new kern.sysclock sysctl node, which will be home to some generic 151 * sysclock configuration variables. Feed-forward clock specific variables will 152 * live under the ffclock subnode. 153 */ 154 155SYSCTL_NODE(_kern, OID_AUTO, sysclock, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 156 "System clock related configuration"); 157SYSCTL_NODE(_kern_sysclock, OID_AUTO, ffclock, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 158 "Feed-forward clock configuration"); 159 160static char *sysclocks[] = {"feedback", "feed-forward"}; 161#define MAX_SYSCLOCK_NAME_LEN 16 162#define NUM_SYSCLOCKS nitems(sysclocks) 163 164static int ffclock_version = 2; 165SYSCTL_INT(_kern_sysclock_ffclock, OID_AUTO, version, CTLFLAG_RD, 166 &ffclock_version, 0, "Feed-forward clock kernel version"); 167 168/* List available sysclocks. */ 169static int 170sysctl_kern_sysclock_available(SYSCTL_HANDLER_ARGS) 171{ 172 struct sbuf *s; 173 int clk, error; 174 175 s = sbuf_new_for_sysctl(NULL, NULL, 176 MAX_SYSCLOCK_NAME_LEN * NUM_SYSCLOCKS, req); 177 if (s == NULL) 178 return (ENOMEM); 179 180 for (clk = 0; clk < NUM_SYSCLOCKS; clk++) { 181 sbuf_cat(s, sysclocks[clk]); 182 if (clk + 1 < NUM_SYSCLOCKS) 183 sbuf_cat(s, " "); 184 } 185 error = sbuf_finish(s); 186 sbuf_delete(s); 187 188 return (error); 189} 190 191SYSCTL_PROC(_kern_sysclock, OID_AUTO, available, 192 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, 0, 0, 193 sysctl_kern_sysclock_available, "A", 194 "List of available system clocks"); 195 196/* 197 * Return the name of the active system clock if read, or attempt to change 198 * the active system clock to the user specified one if written to. The active 199 * system clock is read when calling any of the [get]{bin,nano,micro}[up]time() 200 * functions. 201 */ 202static int 203sysctl_kern_sysclock_active(SYSCTL_HANDLER_ARGS) 204{ 205 char newclock[MAX_SYSCLOCK_NAME_LEN]; 206 int error; 207 int clk; 208 209 /* Return the name of the current active sysclock. */ 210 strlcpy(newclock, sysclocks[sysclock_active], sizeof(newclock)); 211 error = sysctl_handle_string(oidp, newclock, sizeof(newclock), req); 212 213 /* Check for error or no change */ 214 if (error != 0 || req->newptr == NULL) 215 goto done; 216 217 /* Change the active sysclock to the user specified one: */ 218 error = EINVAL; 219 for (clk = 0; clk < NUM_SYSCLOCKS; clk++) { 220 if (strncmp(newclock, sysclocks[clk], 221 MAX_SYSCLOCK_NAME_LEN - 1)) { 222 continue; 223 } 224 sysclock_active = clk; 225 error = 0; 226 break; 227 } 228done: 229 return (error); 230} 231 232SYSCTL_PROC(_kern_sysclock, OID_AUTO, active, 233 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 0, 0, 234 sysctl_kern_sysclock_active, "A", 235 "Name of the active system clock which is currently serving time"); 236 237static int sysctl_kern_ffclock_ffcounter_bypass = 0; 238SYSCTL_INT(_kern_sysclock_ffclock, OID_AUTO, ffcounter_bypass, CTLFLAG_RW, 239 &sysctl_kern_ffclock_ffcounter_bypass, 0, 240 "Use reliable hardware timecounter as the feed-forward counter"); 241 242/* 243 * High level functions to access the Feed-Forward Clock. 244 */ 245void 246ffclock_bintime(struct bintime *bt) 247{ 248 249 ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); 250} 251 252void 253ffclock_nanotime(struct timespec *tsp) 254{ 255 struct bintime bt; 256 257 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); 258 bintime2timespec(&bt, tsp); 259} 260 261void 262ffclock_microtime(struct timeval *tvp) 263{ 264 struct bintime bt; 265 266 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); 267 bintime2timeval(&bt, tvp); 268} 269 270void 271ffclock_getbintime(struct bintime *bt) 272{ 273 274 ffclock_abstime(NULL, bt, NULL, 275 FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); 276} 277 278void 279ffclock_getnanotime(struct timespec *tsp) 280{ 281 struct bintime bt; 282 283 ffclock_abstime(NULL, &bt, NULL, 284 FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); 285 bintime2timespec(&bt, tsp); 286} 287 288void 289ffclock_getmicrotime(struct timeval *tvp) 290{ 291 struct bintime bt; 292 293 ffclock_abstime(NULL, &bt, NULL, 294 FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); 295 bintime2timeval(&bt, tvp); 296} 297 298void 299ffclock_binuptime(struct bintime *bt) 300{ 301 302 ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); 303} 304 305void 306ffclock_nanouptime(struct timespec *tsp) 307{ 308 struct bintime bt; 309 310 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); 311 bintime2timespec(&bt, tsp); 312} 313 314void 315ffclock_microuptime(struct timeval *tvp) 316{ 317 struct bintime bt; 318 319 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); 320 bintime2timeval(&bt, tvp); 321} 322 323void 324ffclock_getbinuptime(struct bintime *bt) 325{ 326 327 ffclock_abstime(NULL, bt, NULL, 328 FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); 329} 330 331void 332ffclock_getnanouptime(struct timespec *tsp) 333{ 334 struct bintime bt; 335 336 ffclock_abstime(NULL, &bt, NULL, 337 FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); 338 bintime2timespec(&bt, tsp); 339} 340 341void 342ffclock_getmicrouptime(struct timeval *tvp) 343{ 344 struct bintime bt; 345 346 ffclock_abstime(NULL, &bt, NULL, 347 FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); 348 bintime2timeval(&bt, tvp); 349} 350 351void 352ffclock_bindifftime(ffcounter ffdelta, struct bintime *bt) 353{ 354 355 ffclock_difftime(ffdelta, bt, NULL); 356} 357 358void 359ffclock_nanodifftime(ffcounter ffdelta, struct timespec *tsp) 360{ 361 struct bintime bt; 362 363 ffclock_difftime(ffdelta, &bt, NULL); 364 bintime2timespec(&bt, tsp); 365} 366 367void 368ffclock_microdifftime(ffcounter ffdelta, struct timeval *tvp) 369{ 370 struct bintime bt; 371 372 ffclock_difftime(ffdelta, &bt, NULL); 373 bintime2timeval(&bt, tvp); 374} 375 376/* 377 * System call allowing userland applications to retrieve the current value of 378 * the Feed-Forward Clock counter. 379 */ 380#ifndef _SYS_SYSPROTO_H_ 381struct ffclock_getcounter_args { 382 ffcounter *ffcount; 383}; 384#endif 385/* ARGSUSED */ 386int 387sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap) 388{ 389 ffcounter ffcount; 390 int error; 391 392 ffcount = 0; 393 ffclock_read_counter(&ffcount); 394 if (ffcount == 0) 395 return (EAGAIN); 396 error = copyout(&ffcount, uap->ffcount, sizeof(ffcounter)); 397 398 return (error); 399} 400 401/* 402 * System call allowing the synchronisation daemon to push new feed-forward clock 403 * estimates to the kernel. Acquire ffclock_mtx to prevent concurrent updates 404 * and ensure data consistency. 405 * NOTE: ffclock_updated signals the fftimehands that new estimates are 406 * available. The updated estimates are picked up by the fftimehands on next 407 * tick, which could take as long as 1/hz seconds (if ticks are not missed). 408 */ 409#ifndef _SYS_SYSPROTO_H_ 410struct ffclock_setestimate_args { 411 struct ffclock_estimate *cest; 412}; 413#endif 414/* ARGSUSED */ 415int 416sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap) 417{ 418 struct ffclock_estimate cest; 419 int error; 420 421 /* Reuse of PRIV_CLOCK_SETTIME. */ 422 if ((error = priv_check(td, PRIV_CLOCK_SETTIME)) != 0) 423 return (error); 424 425 if ((error = copyin(uap->cest, &cest, sizeof(struct ffclock_estimate))) 426 != 0) 427 return (error); 428 429 mtx_lock(&ffclock_mtx); 430 memcpy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate)); 431 ffclock_updated++; 432 mtx_unlock(&ffclock_mtx); 433 return (error); 434} 435 436/* 437 * System call allowing userland applications to retrieve the clock estimates 438 * stored within the kernel. It is useful to kickstart the synchronisation 439 * daemon with the kernel's knowledge of hardware timecounter. 440 */ 441#ifndef _SYS_SYSPROTO_H_ 442struct ffclock_getestimate_args { 443 struct ffclock_estimate *cest; 444}; 445#endif 446/* ARGSUSED */ 447int 448sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap) 449{ 450 struct ffclock_estimate cest; 451 int error; 452 453 mtx_lock(&ffclock_mtx); 454 memcpy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate)); 455 mtx_unlock(&ffclock_mtx); 456 error = copyout(&cest, uap->cest, sizeof(struct ffclock_estimate)); 457 return (error); 458} 459 460#else /* !FFCLOCK */ 461 462int 463sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap) 464{ 465 466 return (ENOSYS); 467} 468 469int 470sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap) 471{ 472 473 return (ENOSYS); 474} 475 476int 477sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap) 478{ 479 480 return (ENOSYS); 481} 482 483#endif /* FFCLOCK */ 484