pvclock.c revision 278183
1278183Sbryanv/*- 2278183Sbryanv * Copyright (c) 2009 Adrian Chadd 3278183Sbryanv * Copyright (c) 2012 Spectra Logic Corporation 4278183Sbryanv * Copyright (c) 2014 Bryan Venteicher 5278183Sbryanv * All rights reserved. 6278183Sbryanv * 7278183Sbryanv * Redistribution and use in source and binary forms, with or without 8278183Sbryanv * modification, are permitted provided that the following conditions 9278183Sbryanv * are met: 10278183Sbryanv * 1. Redistributions of source code must retain the above copyright 11278183Sbryanv * notice, this list of conditions and the following disclaimer. 12278183Sbryanv * 2. Redistributions in binary form must reproduce the above copyright 13278183Sbryanv * notice, this list of conditions and the following disclaimer in the 14278183Sbryanv * documentation and/or other materials provided with the distribution. 15278183Sbryanv * 16278183Sbryanv * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17278183Sbryanv * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18278183Sbryanv * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19278183Sbryanv * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20278183Sbryanv * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21278183Sbryanv * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22278183Sbryanv * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23278183Sbryanv * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24278183Sbryanv * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25278183Sbryanv * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26278183Sbryanv * SUCH DAMAGE. 27278183Sbryanv */ 28278183Sbryanv 29278183Sbryanv#include <sys/cdefs.h> 30278183Sbryanv__FBSDID("$FreeBSD: head/sys/x86/x86/pvclock.c 278183 2015-02-04 08:26:43Z bryanv $"); 31278183Sbryanv 32278183Sbryanv#include <sys/param.h> 33278183Sbryanv#include <sys/systm.h> 34278183Sbryanv#include <sys/proc.h> 35278183Sbryanv 36278183Sbryanv#include <machine/cpufunc.h> 37278183Sbryanv#include <machine/cpu.h> 38278183Sbryanv#include <machine/atomic.h> 39278183Sbryanv#include <machine/pvclock.h> 40278183Sbryanv 41278183Sbryanv/* 42278183Sbryanv * Last time; this guarantees a monotonically increasing clock for when 43278183Sbryanv * a stable TSC is not provided. 44278183Sbryanv */ 45278183Sbryanvstatic volatile uint64_t pvclock_last_cycles; 46278183Sbryanv 47278183Sbryanvvoid 48278183Sbryanvpvclock_resume(void) 49278183Sbryanv{ 50278183Sbryanv 51278183Sbryanv atomic_store_rel_64(&pvclock_last_cycles, 0); 52278183Sbryanv} 53278183Sbryanv 54278183Sbryanvuint64_t 55278183Sbryanvpvclock_get_last_cycles(void) 56278183Sbryanv{ 57278183Sbryanv 58278183Sbryanv return (atomic_load_acq_64(&pvclock_last_cycles)); 59278183Sbryanv} 60278183Sbryanv 61278183Sbryanv/* 62278183Sbryanv * Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction, 63278183Sbryanv * yielding a 64-bit result. 64278183Sbryanv */ 65278183Sbryanvstatic inline uint64_t 66278183Sbryanvpvclock_scale_delta(uint64_t delta, uint32_t mul_frac, int shift) 67278183Sbryanv{ 68278183Sbryanv uint64_t product; 69278183Sbryanv 70278183Sbryanv if (shift < 0) 71278183Sbryanv delta >>= -shift; 72278183Sbryanv else 73278183Sbryanv delta <<= shift; 74278183Sbryanv 75278183Sbryanv#if defined(__i386__) 76278183Sbryanv { 77278183Sbryanv uint32_t tmp1, tmp2; 78278183Sbryanv 79278183Sbryanv /** 80278183Sbryanv * For i386, the formula looks like: 81278183Sbryanv * 82278183Sbryanv * lower = (mul_frac * (delta & UINT_MAX)) >> 32 83278183Sbryanv * upper = mul_frac * (delta >> 32) 84278183Sbryanv * product = lower + upper 85278183Sbryanv */ 86278183Sbryanv __asm__ ( 87278183Sbryanv "mul %5 ; " 88278183Sbryanv "mov %4,%%eax ; " 89278183Sbryanv "mov %%edx,%4 ; " 90278183Sbryanv "mul %5 ; " 91278183Sbryanv "xor %5,%5 ; " 92278183Sbryanv "add %4,%%eax ; " 93278183Sbryanv "adc %5,%%edx ; " 94278183Sbryanv : "=A" (product), "=r" (tmp1), "=r" (tmp2) 95278183Sbryanv : "a" ((uint32_t)delta), "1" ((uint32_t)(delta >> 32)), 96278183Sbryanv "2" (mul_frac) ); 97278183Sbryanv } 98278183Sbryanv#elif defined(__amd64__) 99278183Sbryanv { 100278183Sbryanv unsigned long tmp; 101278183Sbryanv 102278183Sbryanv __asm__ ( 103278183Sbryanv "mulq %[mul_frac] ; shrd $32, %[hi], %[lo]" 104278183Sbryanv : [lo]"=a" (product), [hi]"=d" (tmp) 105278183Sbryanv : "0" (delta), [mul_frac]"rm"((uint64_t)mul_frac)); 106278183Sbryanv } 107278183Sbryanv#else 108278183Sbryanv#error "pvclock: unsupported x86 architecture?" 109278183Sbryanv#endif 110278183Sbryanv 111278183Sbryanv return (product); 112278183Sbryanv} 113278183Sbryanv 114278183Sbryanvstatic uint64_t 115278183Sbryanvpvclock_get_nsec_offset(struct pvclock_vcpu_time_info *ti) 116278183Sbryanv{ 117278183Sbryanv uint64_t delta; 118278183Sbryanv 119278183Sbryanv delta = rdtsc() - ti->tsc_timestamp; 120278183Sbryanv 121278183Sbryanv return (pvclock_scale_delta(delta, ti->tsc_to_system_mul, 122278183Sbryanv ti->tsc_shift)); 123278183Sbryanv} 124278183Sbryanv 125278183Sbryanvstatic void 126278183Sbryanvpvclock_read_time_info(struct pvclock_vcpu_time_info *ti, 127278183Sbryanv uint64_t *cycles, uint8_t *flags) 128278183Sbryanv{ 129278183Sbryanv uint32_t version; 130278183Sbryanv 131278183Sbryanv do { 132278183Sbryanv version = ti->version; 133278183Sbryanv rmb(); 134278183Sbryanv *cycles = ti->system_time + pvclock_get_nsec_offset(ti); 135278183Sbryanv *flags = ti->flags; 136278183Sbryanv rmb(); 137278183Sbryanv } while ((ti->version & 1) != 0 || ti->version != version); 138278183Sbryanv} 139278183Sbryanv 140278183Sbryanvstatic void 141278183Sbryanvpvclock_read_wall_clock(struct pvclock_wall_clock *wc, uint32_t *sec, 142278183Sbryanv uint32_t *nsec) 143278183Sbryanv{ 144278183Sbryanv uint32_t version; 145278183Sbryanv 146278183Sbryanv do { 147278183Sbryanv version = wc->version; 148278183Sbryanv rmb(); 149278183Sbryanv *sec = wc->sec; 150278183Sbryanv *nsec = wc->nsec; 151278183Sbryanv rmb(); 152278183Sbryanv } while ((wc->version & 1) != 0 || wc->version != version); 153278183Sbryanv} 154278183Sbryanv 155278183Sbryanvuint64_t 156278183Sbryanvpvclock_get_timecount(struct pvclock_vcpu_time_info *ti) 157278183Sbryanv{ 158278183Sbryanv uint64_t now, last; 159278183Sbryanv uint8_t flags; 160278183Sbryanv 161278183Sbryanv pvclock_read_time_info(ti, &now, &flags); 162278183Sbryanv 163278183Sbryanv if (flags & PVCLOCK_FLAG_TSC_STABLE) 164278183Sbryanv return (now); 165278183Sbryanv 166278183Sbryanv /* 167278183Sbryanv * Enforce a monotonically increasing clock time across all VCPUs. 168278183Sbryanv * If our time is too old, use the last time and return. Otherwise, 169278183Sbryanv * try to update the last time. 170278183Sbryanv */ 171278183Sbryanv do { 172278183Sbryanv last = atomic_load_acq_64(&pvclock_last_cycles); 173278183Sbryanv if (last > now) 174278183Sbryanv return (last); 175278183Sbryanv } while (!atomic_cmpset_64(&pvclock_last_cycles, last, now)); 176278183Sbryanv 177278183Sbryanv return (now); 178278183Sbryanv} 179278183Sbryanv 180278183Sbryanvvoid 181278183Sbryanvpvclock_get_wallclock(struct pvclock_wall_clock *wc, struct timespec *ts) 182278183Sbryanv{ 183278183Sbryanv uint32_t sec, nsec; 184278183Sbryanv 185278183Sbryanv pvclock_read_wall_clock(wc, &sec, &nsec); 186278183Sbryanv ts->tv_sec = sec; 187278183Sbryanv ts->tv_nsec = nsec; 188278183Sbryanv} 189