11638Srgrimes// SPDX-License-Identifier: GPL-2.0 21638Srgrimes/* 31638Srgrimes * Xen stolen ticks accounting. 41638Srgrimes */ 51638Srgrimes#include <linux/kernel.h> 61638Srgrimes#include <linux/kernel_stat.h> 71638Srgrimes#include <linux/math64.h> 81638Srgrimes#include <linux/gfp.h> 91638Srgrimes#include <linux/slab.h> 101638Srgrimes#include <linux/static_call.h> 111638Srgrimes 121638Srgrimes#include <asm/paravirt.h> 131638Srgrimes#include <asm/xen/hypervisor.h> 141638Srgrimes#include <asm/xen/hypercall.h> 151638Srgrimes 161638Srgrimes#include <xen/events.h> 171638Srgrimes#include <xen/features.h> 181638Srgrimes#include <xen/interface/xen.h> 191638Srgrimes#include <xen/interface/vcpu.h> 201638Srgrimes#include <xen/xen-ops.h> 211638Srgrimes 221638Srgrimes/* runstate info updated by Xen */ 231638Srgrimesstatic DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate); 241638Srgrimes 251638Srgrimesstatic DEFINE_PER_CPU(u64[4], old_runstate_time); 261638Srgrimes 271638Srgrimes/* return an consistent snapshot of 64-bit time/counter value */ 281638Srgrimesstatic u64 get64(const u64 *p) 291638Srgrimes{ 301638Srgrimes u64 ret; 311638Srgrimes 321638Srgrimes if (BITS_PER_LONG < 64) { 331638Srgrimes u32 *p32 = (u32 *)p; 3429625Sache u32 h, l, h2; 3529625Sache 3629625Sache /* 3729625Sache * Read high then low, and then make sure high is 3829625Sache * still the same; this will only loop if low wraps 3929625Sache * and carries into high. 4029625Sache * XXX some clean way to make this endian-proof? 4129625Sache */ 4229625Sache do { 4329625Sache h = READ_ONCE(p32[1]); 4429625Sache l = READ_ONCE(p32[0]); 4529625Sache h2 = READ_ONCE(p32[1]); 4629625Sache } while(h2 != h); 4729625Sache 4829625Sache ret = (((u64)h) << 32) | l; 4929625Sache } else 5029625Sache ret = READ_ONCE(*p); 5129625Sache 5229625Sache return ret; 5329625Sache} 5450476Speter 5529625Sachestatic void xen_get_runstate_snapshot_cpu_delta( 56200503Sdougb struct vcpu_runstate_info *res, unsigned int cpu) 571638Srgrimes{ 5879538Sru u64 state_time; 591638Srgrimes struct vcpu_runstate_info *state; 601638Srgrimes 611638Srgrimes BUG_ON(preemptible()); 621638Srgrimes 6324699Sjmg state = per_cpu_ptr(&xen_runstate, cpu); 641638Srgrimes 651638Srgrimes do { 6624699Sjmg state_time = get64(&state->state_entry_time); 671638Srgrimes rmb(); /* Hypervisor might update data. */ 681638Srgrimes *res = __READ_ONCE(*state); 691638Srgrimes rmb(); /* Hypervisor might update data. */ 701638Srgrimes } while (get64(&state->state_entry_time) != state_time || 711638Srgrimes (state_time & XEN_RUNSTATE_UPDATE)); 721638Srgrimes} 731638Srgrimes 7457665Snikstatic void xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info *res, 751638Srgrimes unsigned int cpu) 7624699Sjmg{ 771638Srgrimes int i; 781638Srgrimes 791638Srgrimes xen_get_runstate_snapshot_cpu_delta(res, cpu); 801638Srgrimes 8168962Sru for (i = 0; i < 4; i++) 821638Srgrimes res->time[i] += per_cpu(old_runstate_time, cpu)[i]; 831638Srgrimes} 8424699Sjmg 851638Srgrimesvoid xen_manage_runstate_time(int action) 861638Srgrimes{ 871638Srgrimes static struct vcpu_runstate_info *runstate_delta; 881638Srgrimes struct vcpu_runstate_info state; 891638Srgrimes int cpu, i; 901638Srgrimes 911638Srgrimes switch (action) { 921638Srgrimes case -1: /* backup runstate time before suspend */ 931638Srgrimes if (unlikely(runstate_delta)) 941638Srgrimes pr_warn_once("%s: memory leak as runstate_delta is not NULL\n", 951638Srgrimes __func__); 961638Srgrimes 971638Srgrimes runstate_delta = kmalloc_array(num_possible_cpus(), 981638Srgrimes sizeof(*runstate_delta), 991638Srgrimes GFP_ATOMIC); 1001638Srgrimes if (unlikely(!runstate_delta)) { 1011638Srgrimes pr_warn("%s: failed to allocate runstate_delta\n", 1021638Srgrimes __func__); 1031638Srgrimes return; 1041638Srgrimes } 1051638Srgrimes 1061638Srgrimes for_each_possible_cpu(cpu) { 1071638Srgrimes xen_get_runstate_snapshot_cpu_delta(&state, cpu); 1081638Srgrimes memcpy(runstate_delta[cpu].time, state.time, 10972126Sru sizeof(runstate_delta[cpu].time)); 11070466Sru } 1111638Srgrimes 1121638Srgrimes break; 1131638Srgrimes 1141638Srgrimes case 0: /* backup runstate time after resume */ 1151638Srgrimes if (unlikely(!runstate_delta)) { 1161638Srgrimes pr_warn("%s: cannot accumulate runstate time as runstate_delta is NULL\n", 1171638Srgrimes __func__); 1181638Srgrimes return; 1191638Srgrimes } 12029625Sache 12157676Ssheldonh for_each_possible_cpu(cpu) { 12257676Ssheldonh for (i = 0; i < 4; i++) 12329625Sache per_cpu(old_runstate_time, cpu)[i] += 12429625Sache runstate_delta[cpu].time[i]; 12529625Sache } 12629625Sache 12729625Sache break; 128116039Scharnier 12929625Sache default: /* do not accumulate runstate time for checkpointing */ 130116039Scharnier break; 13129625Sache } 13229625Sache 13329625Sache if (action != -1 && runstate_delta) { 13429625Sache kfree(runstate_delta); 13529625Sache runstate_delta = NULL; 13629625Sache } 13729625Sache} 13829625Sache 13971263Sru/* 14029625Sache * Runstate accounting 14129625Sache */ 14229625Sachevoid xen_get_runstate_snapshot(struct vcpu_runstate_info *res) 14329625Sache{ 14429625Sache xen_get_runstate_snapshot_cpu(res, smp_processor_id()); 14529625Sache} 14629625Sache 14729625Sache/* return true when a vcpu could run but has no real cpu to run on */ 14829625Sachebool xen_vcpu_stolen(int vcpu) 14929625Sache{ 15029625Sache return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable; 15129625Sache} 15229625Sache 15329625Sacheu64 xen_steal_clock(int cpu) 15429625Sache{ 15529625Sache struct vcpu_runstate_info state; 15629625Sache 15729625Sache xen_get_runstate_snapshot_cpu(&state, cpu); 15829625Sache return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline]; 15929625Sache} 16029625Sache 16129625Sachevoid xen_setup_runstate_info(int cpu) 162141846Sru{ 16329625Sache struct vcpu_register_runstate_memory_area area; 16429625Sache 165141846Sru area.addr.v = &per_cpu(xen_runstate, cpu); 166141846Sru 16729625Sache if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area, 16829625Sache xen_vcpu_nr(cpu), &area)) 16929625Sache BUG(); 17029625Sache} 17129625Sache 17229625Sachevoid __init xen_time_setup_guest(void) 17329625Sache{ 17429625Sache bool xen_runstate_remote; 17529625Sache 17629625Sache xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable, 17729625Sache VMASST_TYPE_runstate_update_flag); 17829625Sache 17929625Sache static_call_update(pv_steal_clock, xen_steal_clock); 18029625Sache 18129625Sache static_key_slow_inc(¶virt_steal_enabled); 18229625Sache if (xen_runstate_remote) 18329625Sache static_key_slow_inc(¶virt_steal_rq_enabled); 18429625Sache} 1851638Srgrimes