1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2012 ARM Ltd. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7#include <linux/cpu.h> 8#include <linux/kvm.h> 9#include <linux/kvm_host.h> 10#include <linux/interrupt.h> 11#include <linux/irq.h> 12#include <linux/irqdomain.h> 13#include <linux/uaccess.h> 14 15#include <clocksource/arm_arch_timer.h> 16#include <asm/arch_timer.h> 17#include <asm/kvm_emulate.h> 18#include <asm/kvm_hyp.h> 19#include <asm/kvm_nested.h> 20 21#include <kvm/arm_vgic.h> 22#include <kvm/arm_arch_timer.h> 23 24#include "trace.h" 25 26static struct timecounter *timecounter; 27static unsigned int host_vtimer_irq; 28static unsigned int host_ptimer_irq; 29static u32 host_vtimer_irq_flags; 30static u32 host_ptimer_irq_flags; 31 32static DEFINE_STATIC_KEY_FALSE(has_gic_active_state); 33 34static const u8 default_ppi[] = { 35 [TIMER_PTIMER] = 30, 36 [TIMER_VTIMER] = 27, 37 [TIMER_HPTIMER] = 26, 38 [TIMER_HVTIMER] = 28, 39}; 40 41static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx); 42static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, 43 struct arch_timer_context *timer_ctx); 44static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx); 45static void kvm_arm_timer_write(struct kvm_vcpu *vcpu, 46 struct arch_timer_context *timer, 47 enum kvm_arch_timer_regs treg, 48 u64 val); 49static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu, 50 struct arch_timer_context *timer, 51 enum kvm_arch_timer_regs treg); 52static bool kvm_arch_timer_get_input_level(int vintid); 53 54static struct irq_ops arch_timer_irq_ops = { 55 .get_input_level = kvm_arch_timer_get_input_level, 56}; 57 58static int nr_timers(struct kvm_vcpu *vcpu) 59{ 60 if (!vcpu_has_nv(vcpu)) 61 return NR_KVM_EL0_TIMERS; 62 63 return NR_KVM_TIMERS; 64} 65 66u32 timer_get_ctl(struct arch_timer_context *ctxt) 67{ 68 struct kvm_vcpu *vcpu = ctxt->vcpu; 69 70 switch(arch_timer_ctx_index(ctxt)) { 71 case TIMER_VTIMER: 72 return __vcpu_sys_reg(vcpu, CNTV_CTL_EL0); 73 case TIMER_PTIMER: 74 return __vcpu_sys_reg(vcpu, CNTP_CTL_EL0); 75 case TIMER_HVTIMER: 76 return __vcpu_sys_reg(vcpu, CNTHV_CTL_EL2); 77 case TIMER_HPTIMER: 78 return __vcpu_sys_reg(vcpu, CNTHP_CTL_EL2); 79 default: 80 WARN_ON(1); 81 return 0; 82 } 83} 84 85u64 timer_get_cval(struct arch_timer_context *ctxt) 86{ 87 struct kvm_vcpu *vcpu = ctxt->vcpu; 88 89 switch(arch_timer_ctx_index(ctxt)) { 90 case TIMER_VTIMER: 91 return __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); 92 case TIMER_PTIMER: 93 return __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0); 94 case TIMER_HVTIMER: 95 return __vcpu_sys_reg(vcpu, CNTHV_CVAL_EL2); 96 case TIMER_HPTIMER: 97 return __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2); 98 default: 99 WARN_ON(1); 100 return 0; 101 } 102} 103 104static u64 timer_get_offset(struct arch_timer_context *ctxt) 105{ 106 u64 offset = 0; 107 108 if (!ctxt) 109 return 0; 110 111 if (ctxt->offset.vm_offset) 112 offset += *ctxt->offset.vm_offset; 113 if (ctxt->offset.vcpu_offset) 114 offset += *ctxt->offset.vcpu_offset; 115 116 return offset; 117} 118 119static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl) 120{ 121 struct kvm_vcpu *vcpu = ctxt->vcpu; 122 123 switch(arch_timer_ctx_index(ctxt)) { 124 case TIMER_VTIMER: 125 __vcpu_sys_reg(vcpu, CNTV_CTL_EL0) = ctl; 126 break; 127 case TIMER_PTIMER: 128 __vcpu_sys_reg(vcpu, CNTP_CTL_EL0) = ctl; 129 break; 130 case TIMER_HVTIMER: 131 __vcpu_sys_reg(vcpu, CNTHV_CTL_EL2) = ctl; 132 break; 133 case TIMER_HPTIMER: 134 __vcpu_sys_reg(vcpu, CNTHP_CTL_EL2) = ctl; 135 break; 136 default: 137 WARN_ON(1); 138 } 139} 140 141static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval) 142{ 143 struct kvm_vcpu *vcpu = ctxt->vcpu; 144 145 switch(arch_timer_ctx_index(ctxt)) { 146 case TIMER_VTIMER: 147 __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0) = cval; 148 break; 149 case TIMER_PTIMER: 150 __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0) = cval; 151 break; 152 case TIMER_HVTIMER: 153 __vcpu_sys_reg(vcpu, CNTHV_CVAL_EL2) = cval; 154 break; 155 case TIMER_HPTIMER: 156 __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2) = cval; 157 break; 158 default: 159 WARN_ON(1); 160 } 161} 162 163static void timer_set_offset(struct arch_timer_context *ctxt, u64 offset) 164{ 165 if (!ctxt->offset.vm_offset) { 166 WARN(offset, "timer %ld\n", arch_timer_ctx_index(ctxt)); 167 return; 168 } 169 170 WRITE_ONCE(*ctxt->offset.vm_offset, offset); 171} 172 173u64 kvm_phys_timer_read(void) 174{ 175 return timecounter->cc->read(timecounter->cc); 176} 177 178void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map) 179{ 180 if (vcpu_has_nv(vcpu)) { 181 if (is_hyp_ctxt(vcpu)) { 182 map->direct_vtimer = vcpu_hvtimer(vcpu); 183 map->direct_ptimer = vcpu_hptimer(vcpu); 184 map->emul_vtimer = vcpu_vtimer(vcpu); 185 map->emul_ptimer = vcpu_ptimer(vcpu); 186 } else { 187 map->direct_vtimer = vcpu_vtimer(vcpu); 188 map->direct_ptimer = vcpu_ptimer(vcpu); 189 map->emul_vtimer = vcpu_hvtimer(vcpu); 190 map->emul_ptimer = vcpu_hptimer(vcpu); 191 } 192 } else if (has_vhe()) { 193 map->direct_vtimer = vcpu_vtimer(vcpu); 194 map->direct_ptimer = vcpu_ptimer(vcpu); 195 map->emul_vtimer = NULL; 196 map->emul_ptimer = NULL; 197 } else { 198 map->direct_vtimer = vcpu_vtimer(vcpu); 199 map->direct_ptimer = NULL; 200 map->emul_vtimer = NULL; 201 map->emul_ptimer = vcpu_ptimer(vcpu); 202 } 203 204 trace_kvm_get_timer_map(vcpu->vcpu_id, map); 205} 206 207static inline bool userspace_irqchip(struct kvm *kvm) 208{ 209 return static_branch_unlikely(&userspace_irqchip_in_use) && 210 unlikely(!irqchip_in_kernel(kvm)); 211} 212 213static void soft_timer_start(struct hrtimer *hrt, u64 ns) 214{ 215 hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns), 216 HRTIMER_MODE_ABS_HARD); 217} 218 219static void soft_timer_cancel(struct hrtimer *hrt) 220{ 221 hrtimer_cancel(hrt); 222} 223 224static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id) 225{ 226 struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id; 227 struct arch_timer_context *ctx; 228 struct timer_map map; 229 230 /* 231 * We may see a timer interrupt after vcpu_put() has been called which 232 * sets the CPU's vcpu pointer to NULL, because even though the timer 233 * has been disabled in timer_save_state(), the hardware interrupt 234 * signal may not have been retired from the interrupt controller yet. 235 */ 236 if (!vcpu) 237 return IRQ_HANDLED; 238 239 get_timer_map(vcpu, &map); 240 241 if (irq == host_vtimer_irq) 242 ctx = map.direct_vtimer; 243 else 244 ctx = map.direct_ptimer; 245 246 if (kvm_timer_should_fire(ctx)) 247 kvm_timer_update_irq(vcpu, true, ctx); 248 249 if (userspace_irqchip(vcpu->kvm) && 250 !static_branch_unlikely(&has_gic_active_state)) 251 disable_percpu_irq(host_vtimer_irq); 252 253 return IRQ_HANDLED; 254} 255 256static u64 kvm_counter_compute_delta(struct arch_timer_context *timer_ctx, 257 u64 val) 258{ 259 u64 now = kvm_phys_timer_read() - timer_get_offset(timer_ctx); 260 261 if (now < val) { 262 u64 ns; 263 264 ns = cyclecounter_cyc2ns(timecounter->cc, 265 val - now, 266 timecounter->mask, 267 &timer_ctx->ns_frac); 268 return ns; 269 } 270 271 return 0; 272} 273 274static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx) 275{ 276 return kvm_counter_compute_delta(timer_ctx, timer_get_cval(timer_ctx)); 277} 278 279static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) 280{ 281 WARN_ON(timer_ctx && timer_ctx->loaded); 282 return timer_ctx && 283 ((timer_get_ctl(timer_ctx) & 284 (ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE); 285} 286 287static bool vcpu_has_wfit_active(struct kvm_vcpu *vcpu) 288{ 289 return (cpus_have_final_cap(ARM64_HAS_WFXT) && 290 vcpu_get_flag(vcpu, IN_WFIT)); 291} 292 293static u64 wfit_delay_ns(struct kvm_vcpu *vcpu) 294{ 295 u64 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu)); 296 struct arch_timer_context *ctx; 297 298 ctx = is_hyp_ctxt(vcpu) ? vcpu_hvtimer(vcpu) : vcpu_vtimer(vcpu); 299 300 return kvm_counter_compute_delta(ctx, val); 301} 302 303/* 304 * Returns the earliest expiration time in ns among guest timers. 305 * Note that it will return 0 if none of timers can fire. 306 */ 307static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu) 308{ 309 u64 min_delta = ULLONG_MAX; 310 int i; 311 312 for (i = 0; i < nr_timers(vcpu); i++) { 313 struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i]; 314 315 WARN(ctx->loaded, "timer %d loaded\n", i); 316 if (kvm_timer_irq_can_fire(ctx)) 317 min_delta = min(min_delta, kvm_timer_compute_delta(ctx)); 318 } 319 320 if (vcpu_has_wfit_active(vcpu)) 321 min_delta = min(min_delta, wfit_delay_ns(vcpu)); 322 323 /* If none of timers can fire, then return 0 */ 324 if (min_delta == ULLONG_MAX) 325 return 0; 326 327 return min_delta; 328} 329 330static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt) 331{ 332 struct arch_timer_cpu *timer; 333 struct kvm_vcpu *vcpu; 334 u64 ns; 335 336 timer = container_of(hrt, struct arch_timer_cpu, bg_timer); 337 vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu); 338 339 /* 340 * Check that the timer has really expired from the guest's 341 * PoV (NTP on the host may have forced it to expire 342 * early). If we should have slept longer, restart it. 343 */ 344 ns = kvm_timer_earliest_exp(vcpu); 345 if (unlikely(ns)) { 346 hrtimer_forward_now(hrt, ns_to_ktime(ns)); 347 return HRTIMER_RESTART; 348 } 349 350 kvm_vcpu_wake_up(vcpu); 351 return HRTIMER_NORESTART; 352} 353 354static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt) 355{ 356 struct arch_timer_context *ctx; 357 struct kvm_vcpu *vcpu; 358 u64 ns; 359 360 ctx = container_of(hrt, struct arch_timer_context, hrtimer); 361 vcpu = ctx->vcpu; 362 363 trace_kvm_timer_hrtimer_expire(ctx); 364 365 /* 366 * Check that the timer has really expired from the guest's 367 * PoV (NTP on the host may have forced it to expire 368 * early). If not ready, schedule for a later time. 369 */ 370 ns = kvm_timer_compute_delta(ctx); 371 if (unlikely(ns)) { 372 hrtimer_forward_now(hrt, ns_to_ktime(ns)); 373 return HRTIMER_RESTART; 374 } 375 376 kvm_timer_update_irq(vcpu, true, ctx); 377 return HRTIMER_NORESTART; 378} 379 380static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx) 381{ 382 enum kvm_arch_timers index; 383 u64 cval, now; 384 385 if (!timer_ctx) 386 return false; 387 388 index = arch_timer_ctx_index(timer_ctx); 389 390 if (timer_ctx->loaded) { 391 u32 cnt_ctl = 0; 392 393 switch (index) { 394 case TIMER_VTIMER: 395 case TIMER_HVTIMER: 396 cnt_ctl = read_sysreg_el0(SYS_CNTV_CTL); 397 break; 398 case TIMER_PTIMER: 399 case TIMER_HPTIMER: 400 cnt_ctl = read_sysreg_el0(SYS_CNTP_CTL); 401 break; 402 case NR_KVM_TIMERS: 403 /* GCC is braindead */ 404 cnt_ctl = 0; 405 break; 406 } 407 408 return (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) && 409 (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) && 410 !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK); 411 } 412 413 if (!kvm_timer_irq_can_fire(timer_ctx)) 414 return false; 415 416 cval = timer_get_cval(timer_ctx); 417 now = kvm_phys_timer_read() - timer_get_offset(timer_ctx); 418 419 return cval <= now; 420} 421 422int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 423{ 424 return vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0; 425} 426 427/* 428 * Reflect the timer output level into the kvm_run structure 429 */ 430void kvm_timer_update_run(struct kvm_vcpu *vcpu) 431{ 432 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); 433 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); 434 struct kvm_sync_regs *regs = &vcpu->run->s.regs; 435 436 /* Populate the device bitmap with the timer states */ 437 regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER | 438 KVM_ARM_DEV_EL1_PTIMER); 439 if (kvm_timer_should_fire(vtimer)) 440 regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER; 441 if (kvm_timer_should_fire(ptimer)) 442 regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER; 443} 444 445static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level, 446 struct arch_timer_context *timer_ctx) 447{ 448 int ret; 449 450 timer_ctx->irq.level = new_level; 451 trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_irq(timer_ctx), 452 timer_ctx->irq.level); 453 454 if (!userspace_irqchip(vcpu->kvm)) { 455 ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu, 456 timer_irq(timer_ctx), 457 timer_ctx->irq.level, 458 timer_ctx); 459 WARN_ON(ret); 460 } 461} 462 463/* Only called for a fully emulated timer */ 464static void timer_emulate(struct arch_timer_context *ctx) 465{ 466 bool should_fire = kvm_timer_should_fire(ctx); 467 468 trace_kvm_timer_emulate(ctx, should_fire); 469 470 if (should_fire != ctx->irq.level) { 471 kvm_timer_update_irq(ctx->vcpu, should_fire, ctx); 472 return; 473 } 474 475 /* 476 * If the timer can fire now, we don't need to have a soft timer 477 * scheduled for the future. If the timer cannot fire at all, 478 * then we also don't need a soft timer. 479 */ 480 if (should_fire || !kvm_timer_irq_can_fire(ctx)) 481 return; 482 483 soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx)); 484} 485 486static void set_cntvoff(u64 cntvoff) 487{ 488 kvm_call_hyp(__kvm_timer_set_cntvoff, cntvoff); 489} 490 491static void set_cntpoff(u64 cntpoff) 492{ 493 if (has_cntpoff()) 494 write_sysreg_s(cntpoff, SYS_CNTPOFF_EL2); 495} 496 497static void timer_save_state(struct arch_timer_context *ctx) 498{ 499 struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu); 500 enum kvm_arch_timers index = arch_timer_ctx_index(ctx); 501 unsigned long flags; 502 503 if (!timer->enabled) 504 return; 505 506 local_irq_save(flags); 507 508 if (!ctx->loaded) 509 goto out; 510 511 switch (index) { 512 u64 cval; 513 514 case TIMER_VTIMER: 515 case TIMER_HVTIMER: 516 timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTV_CTL)); 517 timer_set_cval(ctx, read_sysreg_el0(SYS_CNTV_CVAL)); 518 519 /* Disable the timer */ 520 write_sysreg_el0(0, SYS_CNTV_CTL); 521 isb(); 522 523 /* 524 * The kernel may decide to run userspace after 525 * calling vcpu_put, so we reset cntvoff to 0 to 526 * ensure a consistent read between user accesses to 527 * the virtual counter and kernel access to the 528 * physical counter of non-VHE case. 529 * 530 * For VHE, the virtual counter uses a fixed virtual 531 * offset of zero, so no need to zero CNTVOFF_EL2 532 * register, but this is actually useful when switching 533 * between EL1/vEL2 with NV. 534 * 535 * Do it unconditionally, as this is either unavoidable 536 * or dirt cheap. 537 */ 538 set_cntvoff(0); 539 break; 540 case TIMER_PTIMER: 541 case TIMER_HPTIMER: 542 timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTP_CTL)); 543 cval = read_sysreg_el0(SYS_CNTP_CVAL); 544 545 cval -= timer_get_offset(ctx); 546 547 timer_set_cval(ctx, cval); 548 549 /* Disable the timer */ 550 write_sysreg_el0(0, SYS_CNTP_CTL); 551 isb(); 552 553 set_cntpoff(0); 554 break; 555 case NR_KVM_TIMERS: 556 BUG(); 557 } 558 559 trace_kvm_timer_save_state(ctx); 560 561 ctx->loaded = false; 562out: 563 local_irq_restore(flags); 564} 565 566/* 567 * Schedule the background timer before calling kvm_vcpu_halt, so that this 568 * thread is removed from its waitqueue and made runnable when there's a timer 569 * interrupt to handle. 570 */ 571static void kvm_timer_blocking(struct kvm_vcpu *vcpu) 572{ 573 struct arch_timer_cpu *timer = vcpu_timer(vcpu); 574 struct timer_map map; 575 576 get_timer_map(vcpu, &map); 577 578 /* 579 * If no timers are capable of raising interrupts (disabled or 580 * masked), then there's no more work for us to do. 581 */ 582 if (!kvm_timer_irq_can_fire(map.direct_vtimer) && 583 !kvm_timer_irq_can_fire(map.direct_ptimer) && 584 !kvm_timer_irq_can_fire(map.emul_vtimer) && 585 !kvm_timer_irq_can_fire(map.emul_ptimer) && 586 !vcpu_has_wfit_active(vcpu)) 587 return; 588 589 /* 590 * At least one guest time will expire. Schedule a background timer. 591 * Set the earliest expiration time among the guest timers. 592 */ 593 soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu)); 594} 595 596static void kvm_timer_unblocking(struct kvm_vcpu *vcpu) 597{ 598 struct arch_timer_cpu *timer = vcpu_timer(vcpu); 599 600 soft_timer_cancel(&timer->bg_timer); 601} 602 603static void timer_restore_state(struct arch_timer_context *ctx) 604{ 605 struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu); 606 enum kvm_arch_timers index = arch_timer_ctx_index(ctx); 607 unsigned long flags; 608 609 if (!timer->enabled) 610 return; 611 612 local_irq_save(flags); 613 614 if (ctx->loaded) 615 goto out; 616 617 switch (index) { 618 u64 cval, offset; 619 620 case TIMER_VTIMER: 621 case TIMER_HVTIMER: 622 set_cntvoff(timer_get_offset(ctx)); 623 write_sysreg_el0(timer_get_cval(ctx), SYS_CNTV_CVAL); 624 isb(); 625 write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTV_CTL); 626 break; 627 case TIMER_PTIMER: 628 case TIMER_HPTIMER: 629 cval = timer_get_cval(ctx); 630 offset = timer_get_offset(ctx); 631 set_cntpoff(offset); 632 cval += offset; 633 write_sysreg_el0(cval, SYS_CNTP_CVAL); 634 isb(); 635 write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTP_CTL); 636 break; 637 case NR_KVM_TIMERS: 638 BUG(); 639 } 640 641 trace_kvm_timer_restore_state(ctx); 642 643 ctx->loaded = true; 644out: 645 local_irq_restore(flags); 646} 647 648static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active) 649{ 650 int r; 651 r = irq_set_irqchip_state(ctx->host_timer_irq, IRQCHIP_STATE_ACTIVE, active); 652 WARN_ON(r); 653} 654 655static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx) 656{ 657 struct kvm_vcpu *vcpu = ctx->vcpu; 658 bool phys_active = false; 659 660 /* 661 * Update the timer output so that it is likely to match the 662 * state we're about to restore. If the timer expires between 663 * this point and the register restoration, we'll take the 664 * interrupt anyway. 665 */ 666 kvm_timer_update_irq(ctx->vcpu, kvm_timer_should_fire(ctx), ctx); 667 668 if (irqchip_in_kernel(vcpu->kvm)) 669 phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx)); 670 671 phys_active |= ctx->irq.level; 672 673 set_timer_irq_phys_active(ctx, phys_active); 674} 675 676static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu) 677{ 678 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); 679 680 /* 681 * Update the timer output so that it is likely to match the 682 * state we're about to restore. If the timer expires between 683 * this point and the register restoration, we'll take the 684 * interrupt anyway. 685 */ 686 kvm_timer_update_irq(vcpu, kvm_timer_should_fire(vtimer), vtimer); 687 688 /* 689 * When using a userspace irqchip with the architected timers and a 690 * host interrupt controller that doesn't support an active state, we 691 * must still prevent continuously exiting from the guest, and 692 * therefore mask the physical interrupt by disabling it on the host 693 * interrupt controller when the virtual level is high, such that the 694 * guest can make forward progress. Once we detect the output level 695 * being de-asserted, we unmask the interrupt again so that we exit 696 * from the guest when the timer fires. 697 */ 698 if (vtimer->irq.level) 699 disable_percpu_irq(host_vtimer_irq); 700 else 701 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); 702} 703 704/* If _pred is true, set bit in _set, otherwise set it in _clr */ 705#define assign_clear_set_bit(_pred, _bit, _clr, _set) \ 706 do { \ 707 if (_pred) \ 708 (_set) |= (_bit); \ 709 else \ 710 (_clr) |= (_bit); \ 711 } while (0) 712 713static void kvm_timer_vcpu_load_nested_switch(struct kvm_vcpu *vcpu, 714 struct timer_map *map) 715{ 716 int hw, ret; 717 718 if (!irqchip_in_kernel(vcpu->kvm)) 719 return; 720 721 /* 722 * We only ever unmap the vtimer irq on a VHE system that runs nested 723 * virtualization, in which case we have both a valid emul_vtimer, 724 * emul_ptimer, direct_vtimer, and direct_ptimer. 725 * 726 * Since this is called from kvm_timer_vcpu_load(), a change between 727 * vEL2 and vEL1/0 will have just happened, and the timer_map will 728 * represent this, and therefore we switch the emul/direct mappings 729 * below. 730 */ 731 hw = kvm_vgic_get_map(vcpu, timer_irq(map->direct_vtimer)); 732 if (hw < 0) { 733 kvm_vgic_unmap_phys_irq(vcpu, timer_irq(map->emul_vtimer)); 734 kvm_vgic_unmap_phys_irq(vcpu, timer_irq(map->emul_ptimer)); 735 736 ret = kvm_vgic_map_phys_irq(vcpu, 737 map->direct_vtimer->host_timer_irq, 738 timer_irq(map->direct_vtimer), 739 &arch_timer_irq_ops); 740 WARN_ON_ONCE(ret); 741 ret = kvm_vgic_map_phys_irq(vcpu, 742 map->direct_ptimer->host_timer_irq, 743 timer_irq(map->direct_ptimer), 744 &arch_timer_irq_ops); 745 WARN_ON_ONCE(ret); 746 747 /* 748 * The virtual offset behaviour is "interesting", as it 749 * always applies when HCR_EL2.E2H==0, but only when 750 * accessed from EL1 when HCR_EL2.E2H==1. So make sure we 751 * track E2H when putting the HV timer in "direct" mode. 752 */ 753 if (map->direct_vtimer == vcpu_hvtimer(vcpu)) { 754 struct arch_timer_offset *offs = &map->direct_vtimer->offset; 755 756 if (vcpu_el2_e2h_is_set(vcpu)) 757 offs->vcpu_offset = NULL; 758 else 759 offs->vcpu_offset = &__vcpu_sys_reg(vcpu, CNTVOFF_EL2); 760 } 761 } 762} 763 764static void timer_set_traps(struct kvm_vcpu *vcpu, struct timer_map *map) 765{ 766 bool tpt, tpc; 767 u64 clr, set; 768 769 /* 770 * No trapping gets configured here with nVHE. See 771 * __timer_enable_traps(), which is where the stuff happens. 772 */ 773 if (!has_vhe()) 774 return; 775 776 /* 777 * Our default policy is not to trap anything. As we progress 778 * within this function, reality kicks in and we start adding 779 * traps based on emulation requirements. 780 */ 781 tpt = tpc = false; 782 783 /* 784 * We have two possibility to deal with a physical offset: 785 * 786 * - Either we have CNTPOFF (yay!) or the offset is 0: 787 * we let the guest freely access the HW 788 * 789 * - or neither of these condition apply: 790 * we trap accesses to the HW, but still use it 791 * after correcting the physical offset 792 */ 793 if (!has_cntpoff() && timer_get_offset(map->direct_ptimer)) 794 tpt = tpc = true; 795 796 /* 797 * Apply the enable bits that the guest hypervisor has requested for 798 * its own guest. We can only add traps that wouldn't have been set 799 * above. 800 */ 801 if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) { 802 u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2); 803 804 /* Use the VHE format for mental sanity */ 805 if (!vcpu_el2_e2h_is_set(vcpu)) 806 val = (val & (CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN)) << 10; 807 808 tpt |= !(val & (CNTHCTL_EL1PCEN << 10)); 809 tpc |= !(val & (CNTHCTL_EL1PCTEN << 10)); 810 } 811 812 /* 813 * Now that we have collected our requirements, compute the 814 * trap and enable bits. 815 */ 816 set = 0; 817 clr = 0; 818 819 assign_clear_set_bit(tpt, CNTHCTL_EL1PCEN << 10, set, clr); 820 assign_clear_set_bit(tpc, CNTHCTL_EL1PCTEN << 10, set, clr); 821 822 /* This only happens on VHE, so use the CNTHCTL_EL2 accessor. */ 823 sysreg_clear_set(cnthctl_el2, clr, set); 824} 825 826void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu) 827{ 828 struct arch_timer_cpu *timer = vcpu_timer(vcpu); 829 struct timer_map map; 830 831 if (unlikely(!timer->enabled)) 832 return; 833 834 get_timer_map(vcpu, &map); 835 836 if (static_branch_likely(&has_gic_active_state)) { 837 if (vcpu_has_nv(vcpu)) 838 kvm_timer_vcpu_load_nested_switch(vcpu, &map); 839 840 kvm_timer_vcpu_load_gic(map.direct_vtimer); 841 if (map.direct_ptimer) 842 kvm_timer_vcpu_load_gic(map.direct_ptimer); 843 } else { 844 kvm_timer_vcpu_load_nogic(vcpu); 845 } 846 847 kvm_timer_unblocking(vcpu); 848 849 timer_restore_state(map.direct_vtimer); 850 if (map.direct_ptimer) 851 timer_restore_state(map.direct_ptimer); 852 if (map.emul_vtimer) 853 timer_emulate(map.emul_vtimer); 854 if (map.emul_ptimer) 855 timer_emulate(map.emul_ptimer); 856 857 timer_set_traps(vcpu, &map); 858} 859 860bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu) 861{ 862 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); 863 struct arch_timer_context *ptimer = vcpu_ptimer(vcpu); 864 struct kvm_sync_regs *sregs = &vcpu->run->s.regs; 865 bool vlevel, plevel; 866 867 if (likely(irqchip_in_kernel(vcpu->kvm))) 868 return false; 869 870 vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER; 871 plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER; 872 873 return kvm_timer_should_fire(vtimer) != vlevel || 874 kvm_timer_should_fire(ptimer) != plevel; 875} 876 877void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) 878{ 879 struct arch_timer_cpu *timer = vcpu_timer(vcpu); 880 struct timer_map map; 881 882 if (unlikely(!timer->enabled)) 883 return; 884 885 get_timer_map(vcpu, &map); 886 887 timer_save_state(map.direct_vtimer); 888 if (map.direct_ptimer) 889 timer_save_state(map.direct_ptimer); 890 891 /* 892 * Cancel soft timer emulation, because the only case where we 893 * need it after a vcpu_put is in the context of a sleeping VCPU, and 894 * in that case we already factor in the deadline for the physical 895 * timer when scheduling the bg_timer. 896 * 897 * In any case, we re-schedule the hrtimer for the physical timer when 898 * coming back to the VCPU thread in kvm_timer_vcpu_load(). 899 */ 900 if (map.emul_vtimer) 901 soft_timer_cancel(&map.emul_vtimer->hrtimer); 902 if (map.emul_ptimer) 903 soft_timer_cancel(&map.emul_ptimer->hrtimer); 904 905 if (kvm_vcpu_is_blocking(vcpu)) 906 kvm_timer_blocking(vcpu); 907} 908 909/* 910 * With a userspace irqchip we have to check if the guest de-asserted the 911 * timer and if so, unmask the timer irq signal on the host interrupt 912 * controller to ensure that we see future timer signals. 913 */ 914static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu) 915{ 916 struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); 917 918 if (!kvm_timer_should_fire(vtimer)) { 919 kvm_timer_update_irq(vcpu, false, vtimer); 920 if (static_branch_likely(&has_gic_active_state)) 921 set_timer_irq_phys_active(vtimer, false); 922 else 923 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); 924 } 925} 926 927void kvm_timer_sync_user(struct kvm_vcpu *vcpu) 928{ 929 struct arch_timer_cpu *timer = vcpu_timer(vcpu); 930 931 if (unlikely(!timer->enabled)) 932 return; 933 934 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) 935 unmask_vtimer_irq_user(vcpu); 936} 937 938void kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu) 939{ 940 struct arch_timer_cpu *timer = vcpu_timer(vcpu); 941 struct timer_map map; 942 943 get_timer_map(vcpu, &map); 944 945 /* 946 * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8 947 * and to 0 for ARMv7. We provide an implementation that always 948 * resets the timer to be disabled and unmasked and is compliant with 949 * the ARMv7 architecture. 950 */ 951 for (int i = 0; i < nr_timers(vcpu); i++) 952 timer_set_ctl(vcpu_get_timer(vcpu, i), 0); 953 954 /* 955 * A vcpu running at EL2 is in charge of the offset applied to 956 * the virtual timer, so use the physical VM offset, and point 957 * the vcpu offset to CNTVOFF_EL2. 958 */ 959 if (vcpu_has_nv(vcpu)) { 960 struct arch_timer_offset *offs = &vcpu_vtimer(vcpu)->offset; 961 962 offs->vcpu_offset = &__vcpu_sys_reg(vcpu, CNTVOFF_EL2); 963 offs->vm_offset = &vcpu->kvm->arch.timer_data.poffset; 964 } 965 966 if (timer->enabled) { 967 for (int i = 0; i < nr_timers(vcpu); i++) 968 kvm_timer_update_irq(vcpu, false, 969 vcpu_get_timer(vcpu, i)); 970 971 if (irqchip_in_kernel(vcpu->kvm)) { 972 kvm_vgic_reset_mapped_irq(vcpu, timer_irq(map.direct_vtimer)); 973 if (map.direct_ptimer) 974 kvm_vgic_reset_mapped_irq(vcpu, timer_irq(map.direct_ptimer)); 975 } 976 } 977 978 if (map.emul_vtimer) 979 soft_timer_cancel(&map.emul_vtimer->hrtimer); 980 if (map.emul_ptimer) 981 soft_timer_cancel(&map.emul_ptimer->hrtimer); 982} 983 984static void timer_context_init(struct kvm_vcpu *vcpu, int timerid) 985{ 986 struct arch_timer_context *ctxt = vcpu_get_timer(vcpu, timerid); 987 struct kvm *kvm = vcpu->kvm; 988 989 ctxt->vcpu = vcpu; 990 991 if (timerid == TIMER_VTIMER) 992 ctxt->offset.vm_offset = &kvm->arch.timer_data.voffset; 993 else 994 ctxt->offset.vm_offset = &kvm->arch.timer_data.poffset; 995 996 hrtimer_init(&ctxt->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); 997 ctxt->hrtimer.function = kvm_hrtimer_expire; 998 999 switch (timerid) { 1000 case TIMER_PTIMER: 1001 case TIMER_HPTIMER: 1002 ctxt->host_timer_irq = host_ptimer_irq; 1003 break; 1004 case TIMER_VTIMER: 1005 case TIMER_HVTIMER: 1006 ctxt->host_timer_irq = host_vtimer_irq; 1007 break; 1008 } 1009} 1010 1011void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu) 1012{ 1013 struct arch_timer_cpu *timer = vcpu_timer(vcpu); 1014 1015 for (int i = 0; i < NR_KVM_TIMERS; i++) 1016 timer_context_init(vcpu, i); 1017 1018 /* Synchronize offsets across timers of a VM if not already provided */ 1019 if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) { 1020 timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read()); 1021 timer_set_offset(vcpu_ptimer(vcpu), 0); 1022 } 1023 1024 hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); 1025 timer->bg_timer.function = kvm_bg_timer_expire; 1026} 1027 1028void kvm_timer_init_vm(struct kvm *kvm) 1029{ 1030 for (int i = 0; i < NR_KVM_TIMERS; i++) 1031 kvm->arch.timer_data.ppi[i] = default_ppi[i]; 1032} 1033 1034void kvm_timer_cpu_up(void) 1035{ 1036 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); 1037 if (host_ptimer_irq) 1038 enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags); 1039} 1040 1041void kvm_timer_cpu_down(void) 1042{ 1043 disable_percpu_irq(host_vtimer_irq); 1044 if (host_ptimer_irq) 1045 disable_percpu_irq(host_ptimer_irq); 1046} 1047 1048int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value) 1049{ 1050 struct arch_timer_context *timer; 1051 1052 switch (regid) { 1053 case KVM_REG_ARM_TIMER_CTL: 1054 timer = vcpu_vtimer(vcpu); 1055 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value); 1056 break; 1057 case KVM_REG_ARM_TIMER_CNT: 1058 if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, 1059 &vcpu->kvm->arch.flags)) { 1060 timer = vcpu_vtimer(vcpu); 1061 timer_set_offset(timer, kvm_phys_timer_read() - value); 1062 } 1063 break; 1064 case KVM_REG_ARM_TIMER_CVAL: 1065 timer = vcpu_vtimer(vcpu); 1066 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value); 1067 break; 1068 case KVM_REG_ARM_PTIMER_CTL: 1069 timer = vcpu_ptimer(vcpu); 1070 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value); 1071 break; 1072 case KVM_REG_ARM_PTIMER_CNT: 1073 if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, 1074 &vcpu->kvm->arch.flags)) { 1075 timer = vcpu_ptimer(vcpu); 1076 timer_set_offset(timer, kvm_phys_timer_read() - value); 1077 } 1078 break; 1079 case KVM_REG_ARM_PTIMER_CVAL: 1080 timer = vcpu_ptimer(vcpu); 1081 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value); 1082 break; 1083 1084 default: 1085 return -1; 1086 } 1087 1088 return 0; 1089} 1090 1091static u64 read_timer_ctl(struct arch_timer_context *timer) 1092{ 1093 /* 1094 * Set ISTATUS bit if it's expired. 1095 * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is 1096 * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit 1097 * regardless of ENABLE bit for our implementation convenience. 1098 */ 1099 u32 ctl = timer_get_ctl(timer); 1100 1101 if (!kvm_timer_compute_delta(timer)) 1102 ctl |= ARCH_TIMER_CTRL_IT_STAT; 1103 1104 return ctl; 1105} 1106 1107u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid) 1108{ 1109 switch (regid) { 1110 case KVM_REG_ARM_TIMER_CTL: 1111 return kvm_arm_timer_read(vcpu, 1112 vcpu_vtimer(vcpu), TIMER_REG_CTL); 1113 case KVM_REG_ARM_TIMER_CNT: 1114 return kvm_arm_timer_read(vcpu, 1115 vcpu_vtimer(vcpu), TIMER_REG_CNT); 1116 case KVM_REG_ARM_TIMER_CVAL: 1117 return kvm_arm_timer_read(vcpu, 1118 vcpu_vtimer(vcpu), TIMER_REG_CVAL); 1119 case KVM_REG_ARM_PTIMER_CTL: 1120 return kvm_arm_timer_read(vcpu, 1121 vcpu_ptimer(vcpu), TIMER_REG_CTL); 1122 case KVM_REG_ARM_PTIMER_CNT: 1123 return kvm_arm_timer_read(vcpu, 1124 vcpu_ptimer(vcpu), TIMER_REG_CNT); 1125 case KVM_REG_ARM_PTIMER_CVAL: 1126 return kvm_arm_timer_read(vcpu, 1127 vcpu_ptimer(vcpu), TIMER_REG_CVAL); 1128 } 1129 return (u64)-1; 1130} 1131 1132static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu, 1133 struct arch_timer_context *timer, 1134 enum kvm_arch_timer_regs treg) 1135{ 1136 u64 val; 1137 1138 switch (treg) { 1139 case TIMER_REG_TVAL: 1140 val = timer_get_cval(timer) - kvm_phys_timer_read() + timer_get_offset(timer); 1141 val = lower_32_bits(val); 1142 break; 1143 1144 case TIMER_REG_CTL: 1145 val = read_timer_ctl(timer); 1146 break; 1147 1148 case TIMER_REG_CVAL: 1149 val = timer_get_cval(timer); 1150 break; 1151 1152 case TIMER_REG_CNT: 1153 val = kvm_phys_timer_read() - timer_get_offset(timer); 1154 break; 1155 1156 case TIMER_REG_VOFF: 1157 val = *timer->offset.vcpu_offset; 1158 break; 1159 1160 default: 1161 BUG(); 1162 } 1163 1164 return val; 1165} 1166 1167u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu, 1168 enum kvm_arch_timers tmr, 1169 enum kvm_arch_timer_regs treg) 1170{ 1171 struct arch_timer_context *timer; 1172 struct timer_map map; 1173 u64 val; 1174 1175 get_timer_map(vcpu, &map); 1176 timer = vcpu_get_timer(vcpu, tmr); 1177 1178 if (timer == map.emul_vtimer || timer == map.emul_ptimer) 1179 return kvm_arm_timer_read(vcpu, timer, treg); 1180 1181 preempt_disable(); 1182 timer_save_state(timer); 1183 1184 val = kvm_arm_timer_read(vcpu, timer, treg); 1185 1186 timer_restore_state(timer); 1187 preempt_enable(); 1188 1189 return val; 1190} 1191 1192static void kvm_arm_timer_write(struct kvm_vcpu *vcpu, 1193 struct arch_timer_context *timer, 1194 enum kvm_arch_timer_regs treg, 1195 u64 val) 1196{ 1197 switch (treg) { 1198 case TIMER_REG_TVAL: 1199 timer_set_cval(timer, kvm_phys_timer_read() - timer_get_offset(timer) + (s32)val); 1200 break; 1201 1202 case TIMER_REG_CTL: 1203 timer_set_ctl(timer, val & ~ARCH_TIMER_CTRL_IT_STAT); 1204 break; 1205 1206 case TIMER_REG_CVAL: 1207 timer_set_cval(timer, val); 1208 break; 1209 1210 case TIMER_REG_VOFF: 1211 *timer->offset.vcpu_offset = val; 1212 break; 1213 1214 default: 1215 BUG(); 1216 } 1217} 1218 1219void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu, 1220 enum kvm_arch_timers tmr, 1221 enum kvm_arch_timer_regs treg, 1222 u64 val) 1223{ 1224 struct arch_timer_context *timer; 1225 struct timer_map map; 1226 1227 get_timer_map(vcpu, &map); 1228 timer = vcpu_get_timer(vcpu, tmr); 1229 if (timer == map.emul_vtimer || timer == map.emul_ptimer) { 1230 soft_timer_cancel(&timer->hrtimer); 1231 kvm_arm_timer_write(vcpu, timer, treg, val); 1232 timer_emulate(timer); 1233 } else { 1234 preempt_disable(); 1235 timer_save_state(timer); 1236 kvm_arm_timer_write(vcpu, timer, treg, val); 1237 timer_restore_state(timer); 1238 preempt_enable(); 1239 } 1240} 1241 1242static int timer_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 1243{ 1244 if (vcpu) 1245 irqd_set_forwarded_to_vcpu(d); 1246 else 1247 irqd_clr_forwarded_to_vcpu(d); 1248 1249 return 0; 1250} 1251 1252static int timer_irq_set_irqchip_state(struct irq_data *d, 1253 enum irqchip_irq_state which, bool val) 1254{ 1255 if (which != IRQCHIP_STATE_ACTIVE || !irqd_is_forwarded_to_vcpu(d)) 1256 return irq_chip_set_parent_state(d, which, val); 1257 1258 if (val) 1259 irq_chip_mask_parent(d); 1260 else 1261 irq_chip_unmask_parent(d); 1262 1263 return 0; 1264} 1265 1266static void timer_irq_eoi(struct irq_data *d) 1267{ 1268 if (!irqd_is_forwarded_to_vcpu(d)) 1269 irq_chip_eoi_parent(d); 1270} 1271 1272static void timer_irq_ack(struct irq_data *d) 1273{ 1274 d = d->parent_data; 1275 if (d->chip->irq_ack) 1276 d->chip->irq_ack(d); 1277} 1278 1279static struct irq_chip timer_chip = { 1280 .name = "KVM", 1281 .irq_ack = timer_irq_ack, 1282 .irq_mask = irq_chip_mask_parent, 1283 .irq_unmask = irq_chip_unmask_parent, 1284 .irq_eoi = timer_irq_eoi, 1285 .irq_set_type = irq_chip_set_type_parent, 1286 .irq_set_vcpu_affinity = timer_irq_set_vcpu_affinity, 1287 .irq_set_irqchip_state = timer_irq_set_irqchip_state, 1288}; 1289 1290static int timer_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1291 unsigned int nr_irqs, void *arg) 1292{ 1293 irq_hw_number_t hwirq = (uintptr_t)arg; 1294 1295 return irq_domain_set_hwirq_and_chip(domain, virq, hwirq, 1296 &timer_chip, NULL); 1297} 1298 1299static void timer_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1300 unsigned int nr_irqs) 1301{ 1302} 1303 1304static const struct irq_domain_ops timer_domain_ops = { 1305 .alloc = timer_irq_domain_alloc, 1306 .free = timer_irq_domain_free, 1307}; 1308 1309static void kvm_irq_fixup_flags(unsigned int virq, u32 *flags) 1310{ 1311 *flags = irq_get_trigger_type(virq); 1312 if (*flags != IRQF_TRIGGER_HIGH && *flags != IRQF_TRIGGER_LOW) { 1313 kvm_err("Invalid trigger for timer IRQ%d, assuming level low\n", 1314 virq); 1315 *flags = IRQF_TRIGGER_LOW; 1316 } 1317} 1318 1319static int kvm_irq_init(struct arch_timer_kvm_info *info) 1320{ 1321 struct irq_domain *domain = NULL; 1322 1323 if (info->virtual_irq <= 0) { 1324 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n", 1325 info->virtual_irq); 1326 return -ENODEV; 1327 } 1328 1329 host_vtimer_irq = info->virtual_irq; 1330 kvm_irq_fixup_flags(host_vtimer_irq, &host_vtimer_irq_flags); 1331 1332 if (kvm_vgic_global_state.no_hw_deactivation) { 1333 struct fwnode_handle *fwnode; 1334 struct irq_data *data; 1335 1336 fwnode = irq_domain_alloc_named_fwnode("kvm-timer"); 1337 if (!fwnode) 1338 return -ENOMEM; 1339 1340 /* Assume both vtimer and ptimer in the same parent */ 1341 data = irq_get_irq_data(host_vtimer_irq); 1342 domain = irq_domain_create_hierarchy(data->domain, 0, 1343 NR_KVM_TIMERS, fwnode, 1344 &timer_domain_ops, NULL); 1345 if (!domain) { 1346 irq_domain_free_fwnode(fwnode); 1347 return -ENOMEM; 1348 } 1349 1350 arch_timer_irq_ops.flags |= VGIC_IRQ_SW_RESAMPLE; 1351 WARN_ON(irq_domain_push_irq(domain, host_vtimer_irq, 1352 (void *)TIMER_VTIMER)); 1353 } 1354 1355 if (info->physical_irq > 0) { 1356 host_ptimer_irq = info->physical_irq; 1357 kvm_irq_fixup_flags(host_ptimer_irq, &host_ptimer_irq_flags); 1358 1359 if (domain) 1360 WARN_ON(irq_domain_push_irq(domain, host_ptimer_irq, 1361 (void *)TIMER_PTIMER)); 1362 } 1363 1364 return 0; 1365} 1366 1367int __init kvm_timer_hyp_init(bool has_gic) 1368{ 1369 struct arch_timer_kvm_info *info; 1370 int err; 1371 1372 info = arch_timer_get_kvm_info(); 1373 timecounter = &info->timecounter; 1374 1375 if (!timecounter->cc) { 1376 kvm_err("kvm_arch_timer: uninitialized timecounter\n"); 1377 return -ENODEV; 1378 } 1379 1380 err = kvm_irq_init(info); 1381 if (err) 1382 return err; 1383 1384 /* First, do the virtual EL1 timer irq */ 1385 1386 err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler, 1387 "kvm guest vtimer", kvm_get_running_vcpus()); 1388 if (err) { 1389 kvm_err("kvm_arch_timer: can't request vtimer interrupt %d (%d)\n", 1390 host_vtimer_irq, err); 1391 return err; 1392 } 1393 1394 if (has_gic) { 1395 err = irq_set_vcpu_affinity(host_vtimer_irq, 1396 kvm_get_running_vcpus()); 1397 if (err) { 1398 kvm_err("kvm_arch_timer: error setting vcpu affinity\n"); 1399 goto out_free_vtimer_irq; 1400 } 1401 1402 static_branch_enable(&has_gic_active_state); 1403 } 1404 1405 kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq); 1406 1407 /* Now let's do the physical EL1 timer irq */ 1408 1409 if (info->physical_irq > 0) { 1410 err = request_percpu_irq(host_ptimer_irq, kvm_arch_timer_handler, 1411 "kvm guest ptimer", kvm_get_running_vcpus()); 1412 if (err) { 1413 kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n", 1414 host_ptimer_irq, err); 1415 goto out_free_vtimer_irq; 1416 } 1417 1418 if (has_gic) { 1419 err = irq_set_vcpu_affinity(host_ptimer_irq, 1420 kvm_get_running_vcpus()); 1421 if (err) { 1422 kvm_err("kvm_arch_timer: error setting vcpu affinity\n"); 1423 goto out_free_ptimer_irq; 1424 } 1425 } 1426 1427 kvm_debug("physical timer IRQ%d\n", host_ptimer_irq); 1428 } else if (has_vhe()) { 1429 kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n", 1430 info->physical_irq); 1431 err = -ENODEV; 1432 goto out_free_vtimer_irq; 1433 } 1434 1435 return 0; 1436 1437out_free_ptimer_irq: 1438 if (info->physical_irq > 0) 1439 free_percpu_irq(host_ptimer_irq, kvm_get_running_vcpus()); 1440out_free_vtimer_irq: 1441 free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus()); 1442 return err; 1443} 1444 1445void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu) 1446{ 1447 struct arch_timer_cpu *timer = vcpu_timer(vcpu); 1448 1449 soft_timer_cancel(&timer->bg_timer); 1450} 1451 1452static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu) 1453{ 1454 u32 ppis = 0; 1455 bool valid; 1456 1457 mutex_lock(&vcpu->kvm->arch.config_lock); 1458 1459 for (int i = 0; i < nr_timers(vcpu); i++) { 1460 struct arch_timer_context *ctx; 1461 int irq; 1462 1463 ctx = vcpu_get_timer(vcpu, i); 1464 irq = timer_irq(ctx); 1465 if (kvm_vgic_set_owner(vcpu, irq, ctx)) 1466 break; 1467 1468 /* 1469 * We know by construction that we only have PPIs, so 1470 * all values are less than 32. 1471 */ 1472 ppis |= BIT(irq); 1473 } 1474 1475 valid = hweight32(ppis) == nr_timers(vcpu); 1476 1477 if (valid) 1478 set_bit(KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE, &vcpu->kvm->arch.flags); 1479 1480 mutex_unlock(&vcpu->kvm->arch.config_lock); 1481 1482 return valid; 1483} 1484 1485static bool kvm_arch_timer_get_input_level(int vintid) 1486{ 1487 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 1488 1489 if (WARN(!vcpu, "No vcpu context!\n")) 1490 return false; 1491 1492 for (int i = 0; i < nr_timers(vcpu); i++) { 1493 struct arch_timer_context *ctx; 1494 1495 ctx = vcpu_get_timer(vcpu, i); 1496 if (timer_irq(ctx) == vintid) 1497 return kvm_timer_should_fire(ctx); 1498 } 1499 1500 /* A timer IRQ has fired, but no matching timer was found? */ 1501 WARN_RATELIMIT(1, "timer INTID%d unknown\n", vintid); 1502 1503 return false; 1504} 1505 1506int kvm_timer_enable(struct kvm_vcpu *vcpu) 1507{ 1508 struct arch_timer_cpu *timer = vcpu_timer(vcpu); 1509 struct timer_map map; 1510 int ret; 1511 1512 if (timer->enabled) 1513 return 0; 1514 1515 /* Without a VGIC we do not map virtual IRQs to physical IRQs */ 1516 if (!irqchip_in_kernel(vcpu->kvm)) 1517 goto no_vgic; 1518 1519 /* 1520 * At this stage, we have the guarantee that the vgic is both 1521 * available and initialized. 1522 */ 1523 if (!timer_irqs_are_valid(vcpu)) { 1524 kvm_debug("incorrectly configured timer irqs\n"); 1525 return -EINVAL; 1526 } 1527 1528 get_timer_map(vcpu, &map); 1529 1530 ret = kvm_vgic_map_phys_irq(vcpu, 1531 map.direct_vtimer->host_timer_irq, 1532 timer_irq(map.direct_vtimer), 1533 &arch_timer_irq_ops); 1534 if (ret) 1535 return ret; 1536 1537 if (map.direct_ptimer) { 1538 ret = kvm_vgic_map_phys_irq(vcpu, 1539 map.direct_ptimer->host_timer_irq, 1540 timer_irq(map.direct_ptimer), 1541 &arch_timer_irq_ops); 1542 } 1543 1544 if (ret) 1545 return ret; 1546 1547no_vgic: 1548 timer->enabled = 1; 1549 return 0; 1550} 1551 1552/* If we have CNTPOFF, permanently set ECV to enable it */ 1553void kvm_timer_init_vhe(void) 1554{ 1555 if (cpus_have_final_cap(ARM64_HAS_ECV_CNTPOFF)) 1556 sysreg_clear_set(cnthctl_el2, 0, CNTHCTL_ECV); 1557} 1558 1559int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 1560{ 1561 int __user *uaddr = (int __user *)(long)attr->addr; 1562 int irq, idx, ret = 0; 1563 1564 if (!irqchip_in_kernel(vcpu->kvm)) 1565 return -EINVAL; 1566 1567 if (get_user(irq, uaddr)) 1568 return -EFAULT; 1569 1570 if (!(irq_is_ppi(irq))) 1571 return -EINVAL; 1572 1573 mutex_lock(&vcpu->kvm->arch.config_lock); 1574 1575 if (test_bit(KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE, 1576 &vcpu->kvm->arch.flags)) { 1577 ret = -EBUSY; 1578 goto out; 1579 } 1580 1581 switch (attr->attr) { 1582 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER: 1583 idx = TIMER_VTIMER; 1584 break; 1585 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER: 1586 idx = TIMER_PTIMER; 1587 break; 1588 case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER: 1589 idx = TIMER_HVTIMER; 1590 break; 1591 case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER: 1592 idx = TIMER_HPTIMER; 1593 break; 1594 default: 1595 ret = -ENXIO; 1596 goto out; 1597 } 1598 1599 /* 1600 * We cannot validate the IRQ unicity before we run, so take it at 1601 * face value. The verdict will be given on first vcpu run, for each 1602 * vcpu. Yes this is late. Blame it on the stupid API. 1603 */ 1604 vcpu->kvm->arch.timer_data.ppi[idx] = irq; 1605 1606out: 1607 mutex_unlock(&vcpu->kvm->arch.config_lock); 1608 return ret; 1609} 1610 1611int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 1612{ 1613 int __user *uaddr = (int __user *)(long)attr->addr; 1614 struct arch_timer_context *timer; 1615 int irq; 1616 1617 switch (attr->attr) { 1618 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER: 1619 timer = vcpu_vtimer(vcpu); 1620 break; 1621 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER: 1622 timer = vcpu_ptimer(vcpu); 1623 break; 1624 case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER: 1625 timer = vcpu_hvtimer(vcpu); 1626 break; 1627 case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER: 1628 timer = vcpu_hptimer(vcpu); 1629 break; 1630 default: 1631 return -ENXIO; 1632 } 1633 1634 irq = timer_irq(timer); 1635 return put_user(irq, uaddr); 1636} 1637 1638int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) 1639{ 1640 switch (attr->attr) { 1641 case KVM_ARM_VCPU_TIMER_IRQ_VTIMER: 1642 case KVM_ARM_VCPU_TIMER_IRQ_PTIMER: 1643 case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER: 1644 case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER: 1645 return 0; 1646 } 1647 1648 return -ENXIO; 1649} 1650 1651int kvm_vm_ioctl_set_counter_offset(struct kvm *kvm, 1652 struct kvm_arm_counter_offset *offset) 1653{ 1654 int ret = 0; 1655 1656 if (offset->reserved) 1657 return -EINVAL; 1658 1659 mutex_lock(&kvm->lock); 1660 1661 if (lock_all_vcpus(kvm)) { 1662 set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags); 1663 1664 /* 1665 * If userspace decides to set the offset using this 1666 * API rather than merely restoring the counter 1667 * values, the offset applies to both the virtual and 1668 * physical views. 1669 */ 1670 kvm->arch.timer_data.voffset = offset->counter_offset; 1671 kvm->arch.timer_data.poffset = offset->counter_offset; 1672 1673 unlock_all_vcpus(kvm); 1674 } else { 1675 ret = -EBUSY; 1676 } 1677 1678 mutex_unlock(&kvm->lock); 1679 1680 return ret; 1681} 1682