1278183Sbryanv/*- 2278183Sbryanv * Copyright (c) 2009 Adrian Chadd 3278183Sbryanv * Copyright (c) 2012 Spectra Logic Corporation 4278183Sbryanv * Copyright (c) 2014 Bryan Venteicher 5278183Sbryanv * All rights reserved. 6278183Sbryanv * 7278183Sbryanv * Redistribution and use in source and binary forms, with or without 8278183Sbryanv * modification, are permitted provided that the following conditions 9278183Sbryanv * are met: 10278183Sbryanv * 1. Redistributions of source code must retain the above copyright 11278183Sbryanv * notice, this list of conditions and the following disclaimer. 12278183Sbryanv * 2. Redistributions in binary form must reproduce the above copyright 13278183Sbryanv * notice, this list of conditions and the following disclaimer in the 14278183Sbryanv * documentation and/or other materials provided with the distribution. 15278183Sbryanv * 16278183Sbryanv * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17278183Sbryanv * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18278183Sbryanv * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19278183Sbryanv * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20278183Sbryanv * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21278183Sbryanv * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22278183Sbryanv * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23278183Sbryanv * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24278183Sbryanv * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25278183Sbryanv * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26278183Sbryanv * SUCH DAMAGE. 27278183Sbryanv */ 28278183Sbryanv 29278183Sbryanv#include <sys/cdefs.h> 30278183Sbryanv__FBSDID("$FreeBSD$"); 31278183Sbryanv 32278183Sbryanv#include <sys/param.h> 33278183Sbryanv#include <sys/systm.h> 34278183Sbryanv#include <sys/proc.h> 35278183Sbryanv 36278183Sbryanv#include <machine/cpufunc.h> 37278183Sbryanv#include <machine/cpu.h> 38278183Sbryanv#include <machine/atomic.h> 39278183Sbryanv#include <machine/pvclock.h> 40278183Sbryanv 41278183Sbryanv/* 42278183Sbryanv * Last time; this guarantees a monotonically increasing clock for when 43278183Sbryanv * a stable TSC is not provided. 44278183Sbryanv */ 45278183Sbryanvstatic volatile uint64_t pvclock_last_cycles; 46278183Sbryanv 47278183Sbryanvvoid 48278183Sbryanvpvclock_resume(void) 49278183Sbryanv{ 50278183Sbryanv 51278183Sbryanv atomic_store_rel_64(&pvclock_last_cycles, 0); 52278183Sbryanv} 53278183Sbryanv 54278183Sbryanvuint64_t 55278183Sbryanvpvclock_get_last_cycles(void) 56278183Sbryanv{ 57278183Sbryanv 58278183Sbryanv return (atomic_load_acq_64(&pvclock_last_cycles)); 59278183Sbryanv} 60278183Sbryanv 61278184Sbryanvuint64_t 62278184Sbryanvpvclock_tsc_freq(struct pvclock_vcpu_time_info *ti) 63278184Sbryanv{ 64278184Sbryanv uint64_t freq; 65278184Sbryanv 66278184Sbryanv freq = (1000000000ULL << 32) / ti->tsc_to_system_mul; 67278184Sbryanv 68278184Sbryanv if (ti->tsc_shift < 0) 69278184Sbryanv freq <<= -ti->tsc_shift; 70278184Sbryanv else 71278184Sbryanv freq >>= ti->tsc_shift; 72278184Sbryanv 73278184Sbryanv return (freq); 74278184Sbryanv} 75278184Sbryanv 76278183Sbryanv/* 77278183Sbryanv * Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction, 78278183Sbryanv * yielding a 64-bit result. 79278183Sbryanv */ 80278183Sbryanvstatic inline uint64_t 81278183Sbryanvpvclock_scale_delta(uint64_t delta, uint32_t mul_frac, int shift) 82278183Sbryanv{ 83278183Sbryanv uint64_t product; 84278183Sbryanv 85278183Sbryanv if (shift < 0) 86278183Sbryanv delta >>= -shift; 87278183Sbryanv else 88278183Sbryanv delta <<= shift; 89278183Sbryanv 90278183Sbryanv#if defined(__i386__) 91278183Sbryanv { 92278183Sbryanv uint32_t tmp1, tmp2; 93278183Sbryanv 94278183Sbryanv /** 95278183Sbryanv * For i386, the formula looks like: 96278183Sbryanv * 97278183Sbryanv * lower = (mul_frac * (delta & UINT_MAX)) >> 32 98278183Sbryanv * upper = mul_frac * (delta >> 32) 99278183Sbryanv * product = lower + upper 100278183Sbryanv */ 101278183Sbryanv __asm__ ( 102278183Sbryanv "mul %5 ; " 103278183Sbryanv "mov %4,%%eax ; " 104278183Sbryanv "mov %%edx,%4 ; " 105278183Sbryanv "mul %5 ; " 106278183Sbryanv "xor %5,%5 ; " 107278183Sbryanv "add %4,%%eax ; " 108278183Sbryanv "adc %5,%%edx ; " 109278183Sbryanv : "=A" (product), "=r" (tmp1), "=r" (tmp2) 110278183Sbryanv : "a" ((uint32_t)delta), "1" ((uint32_t)(delta >> 32)), 111278183Sbryanv "2" (mul_frac) ); 112278183Sbryanv } 113278183Sbryanv#elif defined(__amd64__) 114278183Sbryanv { 115278183Sbryanv unsigned long tmp; 116278183Sbryanv 117278183Sbryanv __asm__ ( 118278183Sbryanv "mulq %[mul_frac] ; shrd $32, %[hi], %[lo]" 119278183Sbryanv : [lo]"=a" (product), [hi]"=d" (tmp) 120278183Sbryanv : "0" (delta), [mul_frac]"rm"((uint64_t)mul_frac)); 121278183Sbryanv } 122278183Sbryanv#else 123278183Sbryanv#error "pvclock: unsupported x86 architecture?" 124278183Sbryanv#endif 125278183Sbryanv 126278183Sbryanv return (product); 127278183Sbryanv} 128278183Sbryanv 129278183Sbryanvstatic uint64_t 130278183Sbryanvpvclock_get_nsec_offset(struct pvclock_vcpu_time_info *ti) 131278183Sbryanv{ 132278183Sbryanv uint64_t delta; 133278183Sbryanv 134278183Sbryanv delta = rdtsc() - ti->tsc_timestamp; 135278183Sbryanv 136278183Sbryanv return (pvclock_scale_delta(delta, ti->tsc_to_system_mul, 137278183Sbryanv ti->tsc_shift)); 138278183Sbryanv} 139278183Sbryanv 140278183Sbryanvstatic void 141278183Sbryanvpvclock_read_time_info(struct pvclock_vcpu_time_info *ti, 142278183Sbryanv uint64_t *cycles, uint8_t *flags) 143278183Sbryanv{ 144278183Sbryanv uint32_t version; 145278183Sbryanv 146278183Sbryanv do { 147278183Sbryanv version = ti->version; 148278183Sbryanv rmb(); 149278183Sbryanv *cycles = ti->system_time + pvclock_get_nsec_offset(ti); 150278183Sbryanv *flags = ti->flags; 151278183Sbryanv rmb(); 152278183Sbryanv } while ((ti->version & 1) != 0 || ti->version != version); 153278183Sbryanv} 154278183Sbryanv 155278183Sbryanvstatic void 156278183Sbryanvpvclock_read_wall_clock(struct pvclock_wall_clock *wc, uint32_t *sec, 157278183Sbryanv uint32_t *nsec) 158278183Sbryanv{ 159278183Sbryanv uint32_t version; 160278183Sbryanv 161278183Sbryanv do { 162278183Sbryanv version = wc->version; 163278183Sbryanv rmb(); 164278183Sbryanv *sec = wc->sec; 165278183Sbryanv *nsec = wc->nsec; 166278183Sbryanv rmb(); 167278183Sbryanv } while ((wc->version & 1) != 0 || wc->version != version); 168278183Sbryanv} 169278183Sbryanv 170278183Sbryanvuint64_t 171278183Sbryanvpvclock_get_timecount(struct pvclock_vcpu_time_info *ti) 172278183Sbryanv{ 173278183Sbryanv uint64_t now, last; 174278183Sbryanv uint8_t flags; 175278183Sbryanv 176278183Sbryanv pvclock_read_time_info(ti, &now, &flags); 177278183Sbryanv 178278183Sbryanv if (flags & PVCLOCK_FLAG_TSC_STABLE) 179278183Sbryanv return (now); 180278183Sbryanv 181278183Sbryanv /* 182278183Sbryanv * Enforce a monotonically increasing clock time across all VCPUs. 183278183Sbryanv * If our time is too old, use the last time and return. Otherwise, 184278183Sbryanv * try to update the last time. 185278183Sbryanv */ 186278183Sbryanv do { 187278183Sbryanv last = atomic_load_acq_64(&pvclock_last_cycles); 188278183Sbryanv if (last > now) 189278183Sbryanv return (last); 190278183Sbryanv } while (!atomic_cmpset_64(&pvclock_last_cycles, last, now)); 191278183Sbryanv 192278183Sbryanv return (now); 193278183Sbryanv} 194278183Sbryanv 195278183Sbryanvvoid 196278183Sbryanvpvclock_get_wallclock(struct pvclock_wall_clock *wc, struct timespec *ts) 197278183Sbryanv{ 198278183Sbryanv uint32_t sec, nsec; 199278183Sbryanv 200278183Sbryanv pvclock_read_wall_clock(wc, &sec, &nsec); 201278183Sbryanv ts->tv_sec = sec; 202278183Sbryanv ts->tv_nsec = nsec; 203278183Sbryanv} 204