1301483Ssephe/*- 2322612Ssephe * Copyright (c) 2016-2017 Microsoft Corp. 3301483Ssephe * All rights reserved. 4301483Ssephe * 5301483Ssephe * Redistribution and use in source and binary forms, with or without 6301483Ssephe * modification, are permitted provided that the following conditions 7301483Ssephe * are met: 8301483Ssephe * 1. Redistributions of source code must retain the above copyright 9301483Ssephe * notice unmodified, this list of conditions, and the following 10301483Ssephe * disclaimer. 11301483Ssephe * 2. Redistributions in binary form must reproduce the above copyright 12301483Ssephe * notice, this list of conditions and the following disclaimer in the 13301483Ssephe * documentation and/or other materials provided with the distribution. 14301483Ssephe * 15301483Ssephe * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16301483Ssephe * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17301483Ssephe * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18301483Ssephe * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19301483Ssephe * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20301483Ssephe * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21301483Ssephe * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22301483Ssephe * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23301483Ssephe * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24301483Ssephe * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25301483Ssephe */ 26301483Ssephe 27301483Ssephe#include <sys/cdefs.h> 28301483Ssephe__FBSDID("$FreeBSD: stable/11/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c 322612 2017-08-17 05:09:22Z sephe $"); 29301483Ssephe 30301483Ssephe#include <sys/param.h> 31311376Ssephe#include <sys/conf.h> 32311376Ssephe#include <sys/fcntl.h> 33311376Ssephe#include <sys/kernel.h> 34311376Ssephe#include <sys/systm.h> 35311376Ssephe#include <sys/timetc.h> 36311376Ssephe#include <sys/vdso.h> 37311376Ssephe 38311376Ssephe#include <machine/cpufunc.h> 39311376Ssephe#include <machine/cputypes.h> 40311376Ssephe#include <machine/md_var.h> 41311376Ssephe#include <machine/specialreg.h> 42311376Ssephe 43311376Ssephe#include <vm/vm.h> 44311376Ssephe 45311376Ssephe#include <dev/hyperv/include/hyperv.h> 46311376Ssephe#include <dev/hyperv/include/hyperv_busdma.h> 47301483Ssephe#include <dev/hyperv/vmbus/hyperv_machdep.h> 48311376Ssephe#include <dev/hyperv/vmbus/hyperv_reg.h> 49311376Ssephe#include <dev/hyperv/vmbus/hyperv_var.h> 50301483Ssephe 51311376Ssephestruct hyperv_reftsc_ctx { 52311376Ssephe struct hyperv_reftsc *tsc_ref; 53311376Ssephe struct hyperv_dma tsc_ref_dma; 54311376Ssephe}; 55311376Ssephe 56311376Ssephestatic uint32_t hyperv_tsc_vdso_timehands( 57311376Ssephe struct vdso_timehands *, 58311376Ssephe struct timecounter *); 59311376Ssephe 60311376Ssephestatic d_open_t hyperv_tsc_open; 61311376Ssephestatic d_mmap_t hyperv_tsc_mmap; 62311376Ssephe 63311376Ssephestatic struct timecounter hyperv_tsc_timecounter = { 64311376Ssephe .tc_get_timecount = NULL, /* based on CPU vendor. */ 65311376Ssephe .tc_counter_mask = 0xffffffff, 66311376Ssephe .tc_frequency = HYPERV_TIMER_FREQ, 67311376Ssephe .tc_name = "Hyper-V-TSC", 68311376Ssephe .tc_quality = 3000, 69311376Ssephe .tc_fill_vdso_timehands = hyperv_tsc_vdso_timehands, 70311376Ssephe}; 71311376Ssephe 72311376Ssephestatic struct cdevsw hyperv_tsc_cdevsw = { 73311376Ssephe .d_version = D_VERSION, 74311376Ssephe .d_open = hyperv_tsc_open, 75311376Ssephe .d_mmap = hyperv_tsc_mmap, 76311376Ssephe .d_name = HYPERV_REFTSC_DEVNAME 77311376Ssephe}; 78311376Ssephe 79311376Ssephestatic struct hyperv_reftsc_ctx hyperv_ref_tsc; 80311376Ssephe 81301483Ssepheuint64_t 82301483Ssephehypercall_md(volatile void *hc_addr, uint64_t in_val, 83301483Ssephe uint64_t in_paddr, uint64_t out_paddr) 84301483Ssephe{ 85301483Ssephe uint64_t status; 86301483Ssephe 87301483Ssephe __asm__ __volatile__ ("mov %0, %%r8" : : "r" (out_paddr): "r8"); 88301483Ssephe __asm__ __volatile__ ("call *%3" : "=a" (status) : 89301483Ssephe "c" (in_val), "d" (in_paddr), "m" (hc_addr)); 90301483Ssephe return (status); 91301483Ssephe} 92311376Ssephe 93311376Ssephestatic int 94311376Ssephehyperv_tsc_open(struct cdev *dev __unused, int oflags, int devtype __unused, 95311376Ssephe struct thread *td __unused) 96311376Ssephe{ 97311376Ssephe 98311376Ssephe if (oflags & FWRITE) 99311376Ssephe return (EPERM); 100311376Ssephe return (0); 101311376Ssephe} 102311376Ssephe 103311376Ssephestatic int 104311376Ssephehyperv_tsc_mmap(struct cdev *dev __unused, vm_ooffset_t offset, 105311376Ssephe vm_paddr_t *paddr, int nprot __unused, vm_memattr_t *memattr __unused) 106311376Ssephe{ 107311376Ssephe 108311376Ssephe KASSERT(hyperv_ref_tsc.tsc_ref != NULL, ("reftsc has not been setup")); 109311376Ssephe 110311376Ssephe /* 111311376Ssephe * NOTE: 112311376Ssephe * 'nprot' does not contain information interested to us; 113311376Ssephe * WR-open is blocked by d_open. 114311376Ssephe */ 115311376Ssephe 116311376Ssephe if (offset != 0) 117311376Ssephe return (EOPNOTSUPP); 118311376Ssephe 119311376Ssephe *paddr = hyperv_ref_tsc.tsc_ref_dma.hv_paddr; 120311376Ssephe return (0); 121311376Ssephe} 122311376Ssephe 123311376Ssephestatic uint32_t 124311376Ssephehyperv_tsc_vdso_timehands(struct vdso_timehands *vdso_th, 125311376Ssephe struct timecounter *tc __unused) 126311376Ssephe{ 127311376Ssephe 128311376Ssephe vdso_th->th_algo = VDSO_TH_ALGO_X86_HVTSC; 129311376Ssephe vdso_th->th_x86_shift = 0; 130311376Ssephe vdso_th->th_x86_hpet_idx = 0; 131311376Ssephe bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); 132311376Ssephe return (1); 133311376Ssephe} 134311376Ssephe 135311376Ssephe#define HYPERV_TSC_TIMECOUNT(fence) \ 136314003Ssephestatic uint64_t \ 137314003Ssephehyperv_tc64_tsc_##fence(void) \ 138311376Ssephe{ \ 139311376Ssephe struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref; \ 140311376Ssephe uint32_t seq; \ 141311376Ssephe \ 142311376Ssephe while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) { \ 143311376Ssephe uint64_t disc, ret, tsc; \ 144311376Ssephe uint64_t scale = tsc_ref->tsc_scale; \ 145311376Ssephe int64_t ofs = tsc_ref->tsc_ofs; \ 146311376Ssephe \ 147311376Ssephe fence(); \ 148311376Ssephe tsc = rdtsc(); \ 149311376Ssephe \ 150311376Ssephe /* ret = ((tsc * scale) >> 64) + ofs */ \ 151311376Ssephe __asm__ __volatile__ ("mulq %3" : \ 152311376Ssephe "=d" (ret), "=a" (disc) : \ 153311376Ssephe "a" (tsc), "r" (scale)); \ 154311376Ssephe ret += ofs; \ 155311376Ssephe \ 156311376Ssephe atomic_thread_fence_acq(); \ 157311376Ssephe if (tsc_ref->tsc_seq == seq) \ 158311376Ssephe return (ret); \ 159311376Ssephe \ 160311376Ssephe /* Sequence changed; re-sync. */ \ 161311376Ssephe } \ 162311376Ssephe /* Fallback to the generic timecounter, i.e. rdmsr. */ \ 163311376Ssephe return (rdmsr(MSR_HV_TIME_REF_COUNT)); \ 164311376Ssephe} \ 165314003Ssephe \ 166314003Ssephestatic u_int \ 167314003Ssephehyperv_tsc_timecount_##fence(struct timecounter *tc __unused) \ 168314003Ssephe{ \ 169314003Ssephe \ 170314003Ssephe return (hyperv_tc64_tsc_##fence()); \ 171314003Ssephe} \ 172311376Ssephestruct __hack 173311376Ssephe 174311376SsepheHYPERV_TSC_TIMECOUNT(lfence); 175311376SsepheHYPERV_TSC_TIMECOUNT(mfence); 176311376Ssephe 177311376Ssephestatic void 178311376Ssephehyperv_tsc_tcinit(void *dummy __unused) 179311376Ssephe{ 180314003Ssephe hyperv_tc64_t tc64 = NULL; 181311376Ssephe uint64_t val, orig; 182311376Ssephe 183311376Ssephe if ((hyperv_features & 184311376Ssephe (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC)) != 185311376Ssephe (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC) || 186311376Ssephe (cpu_feature & CPUID_SSE2) == 0) /* SSE2 for mfence/lfence */ 187311376Ssephe return; 188311376Ssephe 189311376Ssephe switch (cpu_vendor_id) { 190311376Ssephe case CPU_VENDOR_AMD: 191311376Ssephe hyperv_tsc_timecounter.tc_get_timecount = 192311376Ssephe hyperv_tsc_timecount_mfence; 193314003Ssephe tc64 = hyperv_tc64_tsc_mfence; 194311376Ssephe break; 195311376Ssephe 196311376Ssephe case CPU_VENDOR_INTEL: 197311376Ssephe hyperv_tsc_timecounter.tc_get_timecount = 198311376Ssephe hyperv_tsc_timecount_lfence; 199314003Ssephe tc64 = hyperv_tc64_tsc_lfence; 200311376Ssephe break; 201311376Ssephe 202311376Ssephe default: 203311376Ssephe /* Unsupport CPU vendors. */ 204311376Ssephe return; 205311376Ssephe } 206311376Ssephe 207311376Ssephe hyperv_ref_tsc.tsc_ref = hyperv_dmamem_alloc(NULL, PAGE_SIZE, 0, 208311376Ssephe sizeof(struct hyperv_reftsc), &hyperv_ref_tsc.tsc_ref_dma, 209311376Ssephe BUS_DMA_WAITOK | BUS_DMA_ZERO); 210311376Ssephe if (hyperv_ref_tsc.tsc_ref == NULL) { 211311376Ssephe printf("hyperv: reftsc page allocation failed\n"); 212311376Ssephe return; 213311376Ssephe } 214311376Ssephe 215311376Ssephe orig = rdmsr(MSR_HV_REFERENCE_TSC); 216311376Ssephe val = MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK) | 217311376Ssephe ((hyperv_ref_tsc.tsc_ref_dma.hv_paddr >> PAGE_SHIFT) << 218311376Ssephe MSR_HV_REFTSC_PGSHIFT); 219311376Ssephe wrmsr(MSR_HV_REFERENCE_TSC, val); 220311376Ssephe 221311376Ssephe /* Register "enlightened" timecounter. */ 222311376Ssephe tc_init(&hyperv_tsc_timecounter); 223311376Ssephe 224314003Ssephe /* Install 64 bits timecounter method for other modules to use. */ 225314003Ssephe KASSERT(tc64 != NULL, ("tc64 is not set")); 226314003Ssephe hyperv_tc64 = tc64; 227314003Ssephe 228311376Ssephe /* Add device for mmap(2). */ 229311376Ssephe make_dev(&hyperv_tsc_cdevsw, 0, UID_ROOT, GID_WHEEL, 0444, 230311376Ssephe HYPERV_REFTSC_DEVNAME); 231311376Ssephe} 232311376SsepheSYSINIT(hyperv_tsc_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, hyperv_tsc_tcinit, 233311376Ssephe NULL); 234