vlapic.c revision 276403
1116742Ssam/*- 2116904Ssam * Copyright (c) 2011 NetApp, Inc. 3186904Ssam * All rights reserved. 4116742Ssam * 5116742Ssam * Redistribution and use in source and binary forms, with or without 6116742Ssam * modification, are permitted provided that the following conditions 7116742Ssam * are met: 8116742Ssam * 1. Redistributions of source code must retain the above copyright 9116742Ssam * notice, this list of conditions and the following disclaimer. 10116904Ssam * 2. Redistributions in binary form must reproduce the above copyright 11116904Ssam * notice, this list of conditions and the following disclaimer in the 12116904Ssam * documentation and/or other materials provided with the distribution. 13116904Ssam * 14116742Ssam * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15116904Ssam * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16116904Ssam * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17116904Ssam * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18116904Ssam * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19116904Ssam * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20116904Ssam * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21116904Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22116904Ssam * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23116904Ssam * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24116904Ssam * SUCH DAMAGE. 25116742Ssam * 26116742Ssam * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 276403 2014-12-30 08:24:14Z neel $ 27116742Ssam */ 28116742Ssam 29116742Ssam#include <sys/cdefs.h> 30116742Ssam__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 276403 2014-12-30 08:24:14Z neel $"); 31193658Ssam 32178354Ssam#include <sys/param.h> 33116742Ssam#include <sys/lock.h> 34116742Ssam#include <sys/kernel.h> 35116742Ssam#include <sys/malloc.h> 36116742Ssam#include <sys/mutex.h> 37116742Ssam#include <sys/systm.h> 38116742Ssam#include <sys/smp.h> 39116742Ssam 40138568Ssam#include <x86/specialreg.h> 41116742Ssam#include <x86/apicreg.h> 42138568Ssam 43138568Ssam#include <machine/clock.h> 44116742Ssam#include <machine/smp.h> 45138568Ssam 46116742Ssam#include <machine/vmm.h> 47138568Ssam 48116742Ssam#include "vmm_ipi.h" 49116742Ssam#include "vmm_lapic.h" 50170530Ssam#include "vmm_ktr.h" 51190391Ssam#include "vmm_stat.h" 52190391Ssam 53190391Ssam#include "vlapic.h" 54186904Ssam#include "vlapic_priv.h" 55186904Ssam#include "vioapic.h" 56186904Ssam 57178354Ssam#define PRIO(x) ((x) >> 4) 58195618Srpaulo 59116742Ssam#define VLAPIC_VERSION (16) 60221022Sbz 61221022Sbz#define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0) 62221022Sbz 63221022Sbz/* 64116742Ssam * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the 65116742Ssam * vlapic_callout_handler() and vcpu accesses to: 66138568Ssam * - timer_freq_bt, timer_period_bt, timer_fire_bt 67138568Ssam * - timer LVT register 68116742Ssam */ 69193658Ssam#define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx)) 70193658Ssam#define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx)) 71193658Ssam#define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx)) 72116742Ssam 73193504Srwatson/* 74193504Srwatson * APIC timer frequency: 75170530Ssam * - arbitrary but chosen to be in the ballpark of contemporary hardware. 76170530Ssam * - power-of-two to avoid loss of precision when converted to a bintime. 77170530Ssam */ 78195618Srpaulo#define VLAPIC_BUS_FREQ (128 * 1024 * 1024) 79195618Srpaulo 80195618Srpaulostatic __inline uint32_t 81195618Srpaulovlapic_get_id(struct vlapic *vlapic) 82195618Srpaulo{ 83195618Srpaulo 84195618Srpaulo if (x2apic(vlapic)) 85195618Srpaulo return (vlapic->vcpuid); 86195618Srpaulo else 87195618Srpaulo return (vlapic->vcpuid << 24); 88195618Srpaulo} 89195618Srpaulo 90178354Ssamstatic uint32_t 91170530Ssamx2apic_ldr(struct vlapic *vlapic) 92170530Ssam{ 93170530Ssam int apicid; 94138568Ssam uint32_t ldr; 95119150Ssam 96138568Ssam apicid = vlapic_get_id(vlapic); 97138568Ssam ldr = 1 << (apicid & 0xf); 98138568Ssam ldr |= (apicid & 0xffff0) << 12; 99138568Ssam return (ldr); 100138568Ssam} 101138568Ssam 102178354Ssamvoid 103138568Ssamvlapic_dfr_write_handler(struct vlapic *vlapic) 104138568Ssam{ 105138568Ssam struct LAPIC *lapic; 106178354Ssam 107138568Ssam lapic = vlapic->apic_page; 108138568Ssam if (x2apic(vlapic)) { 109138568Ssam VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x", 110138568Ssam lapic->dfr); 111138568Ssam lapic->dfr = 0; 112138568Ssam return; 113253727Sadrian } 114253727Sadrian 115253727Sadrian lapic->dfr &= APIC_DFR_MODEL_MASK; 116253727Sadrian lapic->dfr |= APIC_DFR_RESERVED; 117253727Sadrian 118253727Sadrian if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT) 119253727Sadrian VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model"); 120253727Sadrian else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER) 121253727Sadrian VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model"); 122253727Sadrian else 123253727Sadrian VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr); 124253727Sadrian} 125253727Sadrian 126253727Sadrianvoid 127253727Sadrianvlapic_ldr_write_handler(struct vlapic *vlapic) 128253727Sadrian{ 129253727Sadrian struct LAPIC *lapic; 130253727Sadrian 131253727Sadrian lapic = vlapic->apic_page; 132253727Sadrian 133253727Sadrian /* LDR is read-only in x2apic mode */ 134283855Sae if (x2apic(vlapic)) { 135253727Sadrian VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x", 136253727Sadrian lapic->ldr); 137253727Sadrian lapic->ldr = x2apic_ldr(vlapic); 138253727Sadrian } else { 139253727Sadrian lapic->ldr &= ~APIC_LDR_RESERVED; 140253727Sadrian VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr); 141253727Sadrian } 142253727Sadrian} 143253727Sadrian 144283855Saevoid 145283855Saevlapic_id_write_handler(struct vlapic *vlapic) 146253727Sadrian{ 147259172Sgavin struct LAPIC *lapic; 148259172Sgavin 149259172Sgavin /* 150259172Sgavin * We don't allow the ID register to be modified so reset it back to 151259172Sgavin * its default value. 152259172Sgavin */ 153253727Sadrian lapic = vlapic->apic_page; 154253727Sadrian lapic->id = vlapic_get_id(vlapic); 155253727Sadrian} 156253727Sadrian 157253727Sadrianstatic int 158253727Sadrianvlapic_timer_divisor(uint32_t dcr) 159253727Sadrian{ 160253727Sadrian switch (dcr & 0xB) { 161253727Sadrian case APIC_TDCR_1: 162253727Sadrian return (1); 163259172Sgavin case APIC_TDCR_2: 164253727Sadrian return (2); 165259172Sgavin case APIC_TDCR_4: 166253727Sadrian return (4); 167253727Sadrian case APIC_TDCR_8: 168253727Sadrian return (8); 169253727Sadrian case APIC_TDCR_16: 170253727Sadrian return (16); 171253727Sadrian case APIC_TDCR_32: 172253727Sadrian return (32); 173253727Sadrian case APIC_TDCR_64: 174283855Sae return (64); 175283855Sae case APIC_TDCR_128: 176253727Sadrian return (128); 177253727Sadrian default: 178253727Sadrian panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr); 179253727Sadrian } 180253727Sadrian} 181253727Sadrian 182253727Sadrian#if 0 183253727Sadrianstatic inline void 184253727Sadrianvlapic_dump_lvt(uint32_t offset, uint32_t *lvt) 185253727Sadrian{ 186253727Sadrian printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset, 187253727Sadrian *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS, 188253727Sadrian *lvt & APIC_LVTT_M); 189253727Sadrian} 190253727Sadrian#endif 191253727Sadrian 192253727Sadrianstatic uint32_t 193253727Sadrianvlapic_get_ccr(struct vlapic *vlapic) 194253727Sadrian{ 195253727Sadrian struct bintime bt_now, bt_rem; 196253727Sadrian struct LAPIC *lapic; 197253727Sadrian uint32_t ccr; 198253727Sadrian 199253727Sadrian ccr = 0; 200253727Sadrian lapic = vlapic->apic_page; 201253727Sadrian 202253727Sadrian VLAPIC_TIMER_LOCK(vlapic); 203253727Sadrian if (callout_active(&vlapic->callout)) { 204253727Sadrian /* 205253727Sadrian * If the timer is scheduled to expire in the future then 206253727Sadrian * compute the value of 'ccr' based on the remaining time. 207253727Sadrian */ 208253727Sadrian binuptime(&bt_now); 209253727Sadrian if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) { 210253727Sadrian bt_rem = vlapic->timer_fire_bt; 211253727Sadrian bintime_sub(&bt_rem, &bt_now); 212253727Sadrian ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt); 213253727Sadrian ccr += bt_rem.frac / vlapic->timer_freq_bt.frac; 214253727Sadrian } 215253727Sadrian } 216253727Sadrian KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, " 217253727Sadrian "icr_timer is %#x", ccr, lapic->icr_timer)); 218253727Sadrian VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x", 219253727Sadrian ccr, lapic->icr_timer); 220254500Sadrian VLAPIC_TIMER_UNLOCK(vlapic); 221253727Sadrian return (ccr); 222253727Sadrian} 223253727Sadrian 224253727Sadrianvoid 225253727Sadrianvlapic_dcr_write_handler(struct vlapic *vlapic) 226253727Sadrian{ 227253727Sadrian struct LAPIC *lapic; 228253727Sadrian int divisor; 229253727Sadrian 230253727Sadrian lapic = vlapic->apic_page; 231253727Sadrian VLAPIC_TIMER_LOCK(vlapic); 232253727Sadrian 233253727Sadrian divisor = vlapic_timer_divisor(lapic->dcr_timer); 234253727Sadrian VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d", 235253727Sadrian lapic->dcr_timer, divisor); 236253727Sadrian 237253727Sadrian /* 238253727Sadrian * Update the timer frequency and the timer period. 239253727Sadrian * 240253727Sadrian * XXX changes to the frequency divider will not take effect until 241283855Sae * the timer is reloaded. 242253727Sadrian */ 243253727Sadrian FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt); 244253727Sadrian vlapic->timer_period_bt = vlapic->timer_freq_bt; 245254082Sadrian bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer); 246253727Sadrian 247253727Sadrian VLAPIC_TIMER_UNLOCK(vlapic); 248253727Sadrian} 249253727Sadrian 250253727Sadrianvoid 251253727Sadrianvlapic_esr_write_handler(struct vlapic *vlapic) 252253727Sadrian{ 253253727Sadrian struct LAPIC *lapic; 254253727Sadrian 255283855Sae lapic = vlapic->apic_page; 256253727Sadrian lapic->esr = vlapic->esr_pending; 257253727Sadrian vlapic->esr_pending = 0; 258283855Sae} 259283855Sae 260253727Sadrianint 261253727Sadrianvlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level) 262253727Sadrian{ 263253727Sadrian struct LAPIC *lapic; 264253727Sadrian uint32_t *irrptr, *tmrptr, mask; 265253727Sadrian int idx; 266253727Sadrian 267253727Sadrian KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector)); 268253727Sadrian 269248069Sadrian lapic = vlapic->apic_page; 270248069Sadrian if (!(lapic->svr & APIC_SVR_ENABLE)) { 271248069Sadrian VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring " 272248069Sadrian "interrupt %d", vector); 273248069Sadrian return (0); 274248069Sadrian } 275248069Sadrian 276248069Sadrian if (vector < 16) { 277248069Sadrian vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR); 278248069Sadrian VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d", 279248069Sadrian vector); 280248069Sadrian return (1); 281248069Sadrian } 282248069Sadrian 283248069Sadrian if (vlapic->ops.set_intr_ready) 284248069Sadrian return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level)); 285248069Sadrian 286248069Sadrian idx = (vector / 32) * 4; 287248069Sadrian mask = 1 << (vector % 32); 288248069Sadrian 289248069Sadrian irrptr = &lapic->irr0; 290248069Sadrian atomic_set_int(&irrptr[idx], mask); 291248069Sadrian 292248069Sadrian /* 293248069Sadrian * Verify that the trigger-mode of the interrupt matches with 294248069Sadrian * the vlapic TMR registers. 295248069Sadrian */ 296248069Sadrian tmrptr = &lapic->tmr0; 297248069Sadrian if ((tmrptr[idx] & mask) != (level ? mask : 0)) { 298248069Sadrian VLAPIC_CTR3(vlapic, "vlapic TMR[%d] is 0x%08x but " 299248069Sadrian "interrupt is %s-triggered", idx / 4, tmrptr[idx], 300248069Sadrian level ? "level" : "edge"); 301248069Sadrian } 302248069Sadrian 303248069Sadrian VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready"); 304248069Sadrian return (1); 305248069Sadrian} 306248069Sadrian 307248069Sadrianstatic __inline uint32_t * 308248069Sadrianvlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset) 309248069Sadrian{ 310248069Sadrian struct LAPIC *lapic = vlapic->apic_page; 311248069Sadrian int i; 312248069Sadrian 313248069Sadrian switch (offset) { 314248069Sadrian case APIC_OFFSET_CMCI_LVT: 315248069Sadrian return (&lapic->lvt_cmci); 316248069Sadrian case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 317248069Sadrian i = (offset - APIC_OFFSET_TIMER_LVT) >> 2; 318248069Sadrian return ((&lapic->lvt_timer) + i);; 319248069Sadrian default: 320248069Sadrian panic("vlapic_get_lvt: invalid LVT\n"); 321248069Sadrian } 322248069Sadrian} 323283855Sae 324248069Sadrianstatic __inline int 325248069Sadrianlvt_off_to_idx(uint32_t offset) 326248069Sadrian{ 327248069Sadrian int index; 328248069Sadrian 329248069Sadrian switch (offset) { 330248069Sadrian case APIC_OFFSET_CMCI_LVT: 331248069Sadrian index = APIC_LVT_CMCI; 332248069Sadrian break; 333248069Sadrian case APIC_OFFSET_TIMER_LVT: 334248069Sadrian index = APIC_LVT_TIMER; 335248069Sadrian break; 336248069Sadrian case APIC_OFFSET_THERM_LVT: 337248069Sadrian index = APIC_LVT_THERMAL; 338248069Sadrian break; 339248069Sadrian case APIC_OFFSET_PERF_LVT: 340248069Sadrian index = APIC_LVT_PMC; 341248069Sadrian break; 342248069Sadrian case APIC_OFFSET_LINT0_LVT: 343248069Sadrian index = APIC_LVT_LINT0; 344248069Sadrian break; 345248069Sadrian case APIC_OFFSET_LINT1_LVT: 346248069Sadrian index = APIC_LVT_LINT1; 347248069Sadrian break; 348248069Sadrian case APIC_OFFSET_ERROR_LVT: 349248069Sadrian index = APIC_LVT_ERROR; 350248069Sadrian break; 351248069Sadrian default: 352248069Sadrian index = -1; 353248069Sadrian break; 354248069Sadrian } 355248069Sadrian KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: " 356248069Sadrian "invalid lvt index %d for offset %#x", index, offset)); 357248069Sadrian 358248069Sadrian return (index); 359248069Sadrian} 360248069Sadrian 361248069Sadrianstatic __inline uint32_t 362248069Sadrianvlapic_get_lvt(struct vlapic *vlapic, uint32_t offset) 363248069Sadrian{ 364248069Sadrian int idx; 365248069Sadrian uint32_t val; 366248069Sadrian 367248069Sadrian idx = lvt_off_to_idx(offset); 368248069Sadrian val = atomic_load_acq_32(&vlapic->lvt_last[idx]); 369248069Sadrian return (val); 370248069Sadrian} 371248069Sadrian 372248069Sadrianvoid 373248069Sadrianvlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset) 374248069Sadrian{ 375248069Sadrian uint32_t *lvtptr, mask, val; 376248069Sadrian struct LAPIC *lapic; 377248069Sadrian int idx; 378248069Sadrian 379248069Sadrian lapic = vlapic->apic_page; 380248069Sadrian lvtptr = vlapic_get_lvtptr(vlapic, offset); 381248069Sadrian val = *lvtptr; 382248069Sadrian idx = lvt_off_to_idx(offset); 383248069Sadrian 384248069Sadrian if (!(lapic->svr & APIC_SVR_ENABLE)) 385248069Sadrian val |= APIC_LVT_M; 386248069Sadrian mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR; 387248069Sadrian switch (offset) { 388248069Sadrian case APIC_OFFSET_TIMER_LVT: 389248069Sadrian mask |= APIC_LVTT_TM; 390248069Sadrian break; 391248069Sadrian case APIC_OFFSET_ERROR_LVT: 392248069Sadrian break; 393248069Sadrian case APIC_OFFSET_LINT0_LVT: 394248069Sadrian case APIC_OFFSET_LINT1_LVT: 395248069Sadrian mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP; 396248069Sadrian /* FALLTHROUGH */ 397253727Sadrian default: 398248069Sadrian mask |= APIC_LVT_DM; 399253727Sadrian break; 400253727Sadrian } 401248069Sadrian val &= mask; 402248069Sadrian *lvtptr = val; 403248069Sadrian atomic_store_rel_32(&vlapic->lvt_last[idx], val); 404248069Sadrian} 405248069Sadrian 406178354Ssamstatic void 407178354Ssamvlapic_mask_lvts(struct vlapic *vlapic) 408178354Ssam{ 409254196Sadrian struct LAPIC *lapic = vlapic->apic_page; 410254196Sadrian 411254196Sadrian lapic->lvt_cmci |= APIC_LVT_M; 412178354Ssam vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT); 413254082Sadrian 414254082Sadrian lapic->lvt_timer |= APIC_LVT_M; 415178354Ssam vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT); 416178354Ssam 417178354Ssam lapic->lvt_thermal |= APIC_LVT_M; 418178354Ssam vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT); 419178354Ssam 420178354Ssam lapic->lvt_pcint |= APIC_LVT_M; 421178354Ssam vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT); 422178354Ssam 423178354Ssam lapic->lvt_lint0 |= APIC_LVT_M; 424178354Ssam vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT); 425254196Sadrian 426283855Sae lapic->lvt_lint1 |= APIC_LVT_M; 427283855Sae vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT); 428178354Ssam 429178354Ssam lapic->lvt_error |= APIC_LVT_M; 430178354Ssam vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT); 431178354Ssam} 432178354Ssam 433178354Ssamstatic int 434254196Sadrianvlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt) 435254082Sadrian{ 436178354Ssam uint32_t vec, mode; 437178354Ssam 438178354Ssam if (lvt & APIC_LVT_M) 439178354Ssam return (0); 440178354Ssam 441178354Ssam vec = lvt & APIC_LVT_VECTOR; 442178354Ssam mode = lvt & APIC_LVT_DM; 443178354Ssam 444178354Ssam switch (mode) { 445178354Ssam case APIC_LVT_DM_FIXED: 446178354Ssam if (vec < 16) { 447178354Ssam vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR); 448178354Ssam return (0); 449178354Ssam } 450178354Ssam if (vlapic_set_intr_ready(vlapic, vec, false)) 451232097Sadrian vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true); 452178354Ssam break; 453254196Sadrian case APIC_LVT_DM_NMI: 454283855Sae vm_inject_nmi(vlapic->vm, vlapic->vcpuid); 455283855Sae break; 456178354Ssam case APIC_LVT_DM_EXTINT: 457178354Ssam vm_inject_extint(vlapic->vm, vlapic->vcpuid); 458178354Ssam break; 459248069Sadrian default: 460254082Sadrian // Other modes ignored 461254082Sadrian return (0); 462254082Sadrian } 463254082Sadrian return (1); 464254082Sadrian} 465254082Sadrian 466254082Sadrian#if 1 467254082Sadrianstatic void 468254082Sadriandump_isrvec_stk(struct vlapic *vlapic) 469254082Sadrian{ 470254082Sadrian int i; 471254082Sadrian uint32_t *isrptr; 472254082Sadrian 473254082Sadrian isrptr = &vlapic->apic_page->isr0; 474254082Sadrian for (i = 0; i < 8; i++) 475254082Sadrian printf("ISR%d 0x%08x\n", i, isrptr[i * 4]); 476248069Sadrian 477178354Ssam for (i = 0; i <= vlapic->isrvec_stk_top; i++) 478254082Sadrian printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]); 479254082Sadrian} 480254082Sadrian#endif 481254082Sadrian 482254082Sadrian/* 483254082Sadrian * Algorithm adopted from section "Interrupt, Task and Processor Priority" 484254082Sadrian * in Intel Architecture Manual Vol 3a. 485248069Sadrian */ 486248069Sadrianstatic void 487248069Sadrianvlapic_update_ppr(struct vlapic *vlapic) 488248069Sadrian{ 489248069Sadrian int isrvec, tpr, ppr; 490248069Sadrian 491248069Sadrian /* 492248069Sadrian * Note that the value on the stack at index 0 is always 0. 493191553Ssam * 494248069Sadrian * This is a placeholder for the value of ISRV when none of the 495178354Ssam * bits is set in the ISRx registers. 496178354Ssam */ 497178354Ssam isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top]; 498178354Ssam tpr = vlapic->apic_page->tpr; 499178354Ssam 500191538Ssam#if 1 501178354Ssam { 502254076Sadrian int i, lastprio, curprio, vector, idx; 503178354Ssam uint32_t *isrptr; 504178354Ssam 505249925Sglebius if (vlapic->isrvec_stk_top == 0 && isrvec != 0) 506254076Sadrian panic("isrvec_stk is corrupted: %d", isrvec); 507254076Sadrian 508254076Sadrian /* 509254076Sadrian * Make sure that the priority of the nested interrupts is 510254076Sadrian * always increasing. 511178354Ssam */ 512178354Ssam lastprio = -1; 513178354Ssam for (i = 1; i <= vlapic->isrvec_stk_top; i++) { 514195849Ssam curprio = PRIO(vlapic->isrvec_stk[i]); 515178354Ssam if (curprio <= lastprio) { 516248069Sadrian dump_isrvec_stk(vlapic); 517178354Ssam panic("isrvec_stk does not satisfy invariant"); 518248069Sadrian } 519178354Ssam lastprio = curprio; 520195849Ssam } 521195849Ssam 522195849Ssam /* 523195849Ssam * Make sure that each bit set in the ISRx registers has a 524195849Ssam * corresponding entry on the isrvec stack. 525195849Ssam */ 526195849Ssam i = 1; 527195849Ssam isrptr = &vlapic->apic_page->isr0; 528195849Ssam for (vector = 0; vector < 256; vector++) { 529195849Ssam idx = (vector / 32) * 4; 530195849Ssam if (isrptr[idx] & (1 << (vector % 32))) { 531195849Ssam if (i > vlapic->isrvec_stk_top || 532248069Sadrian vlapic->isrvec_stk[i] != vector) { 533178354Ssam dump_isrvec_stk(vlapic); 534178354Ssam panic("ISR and isrvec_stk out of sync"); 535178354Ssam } 536178354Ssam i++; 537178354Ssam } 538191148Skmacy } 539178354Ssam } 540193504Srwatson#endif 541178354Ssam 542178354Ssam if (PRIO(tpr) >= PRIO(isrvec)) 543178354Ssam ppr = tpr; 544178354Ssam else 545178354Ssam ppr = isrvec & 0xf0; 546178354Ssam 547178354Ssam vlapic->apic_page->ppr = ppr; 548178354Ssam VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr); 549178354Ssam} 550178354Ssam 551178354Ssamstatic void 552178354Ssamvlapic_process_eoi(struct vlapic *vlapic) 553178354Ssam{ 554219604Sbschmidt struct LAPIC *lapic = vlapic->apic_page; 555219604Sbschmidt uint32_t *isrptr, *tmrptr; 556178354Ssam int i, idx, bitpos, vector; 557178354Ssam 558178354Ssam isrptr = &lapic->isr0; 559178354Ssam tmrptr = &lapic->tmr0; 560178354Ssam 561178354Ssam /* 562178354Ssam * The x86 architecture reserves the the first 32 vectors for use 563178354Ssam * by the processor. 564178354Ssam */ 565178354Ssam for (i = 7; i > 0; i--) { 566178354Ssam idx = i * 4; 567178354Ssam bitpos = fls(isrptr[idx]); 568178354Ssam if (bitpos-- != 0) { 569178354Ssam if (vlapic->isrvec_stk_top <= 0) { 570178354Ssam panic("invalid vlapic isrvec_stk_top %d", 571178354Ssam vlapic->isrvec_stk_top); 572178354Ssam } 573178354Ssam isrptr[idx] &= ~(1 << bitpos); 574178354Ssam VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi"); 575178354Ssam vlapic->isrvec_stk_top--; 576178354Ssam vlapic_update_ppr(vlapic); 577178354Ssam if ((tmrptr[idx] & (1 << bitpos)) != 0) { 578178354Ssam vector = i * 32 + bitpos; 579178354Ssam vioapic_process_eoi(vlapic->vm, vlapic->vcpuid, 580178354Ssam vector); 581178354Ssam } 582178354Ssam return; 583178354Ssam } 584178354Ssam } 585178354Ssam} 586178354Ssam 587178354Ssamstatic __inline int 588178354Ssamvlapic_get_lvt_field(uint32_t lvt, uint32_t mask) 589178354Ssam{ 590178354Ssam 591178354Ssam return (lvt & mask); 592178354Ssam} 593178354Ssam 594178354Ssamstatic __inline int 595178354Ssamvlapic_periodic_timer(struct vlapic *vlapic) 596178354Ssam{ 597178354Ssam uint32_t lvt; 598178354Ssam 599178354Ssam lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT); 600178354Ssam 601178354Ssam return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC)); 602178354Ssam} 603195845Ssam 604191536Ssamstatic VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic"); 605191536Ssam 606191536Ssamvoid 607191536Ssamvlapic_set_error(struct vlapic *vlapic, uint32_t mask) 608191536Ssam{ 609191536Ssam uint32_t lvt; 610191536Ssam 611191536Ssam vlapic->esr_pending |= mask; 612191536Ssam if (vlapic->esr_firing) 613248069Sadrian return; 614248069Sadrian vlapic->esr_firing = 1; 615178354Ssam 616178354Ssam // The error LVT always uses the fixed delivery mode. 617178354Ssam lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT); 618178354Ssam if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) { 619178354Ssam vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1); 620178354Ssam } 621248069Sadrian vlapic->esr_firing = 0; 622178354Ssam} 623178354Ssam 624248069Sadrianstatic VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic"); 625248069Sadrian 626178354Ssamstatic void 627178354Ssamvlapic_fire_timer(struct vlapic *vlapic) 628178354Ssam{ 629178354Ssam uint32_t lvt; 630178354Ssam 631195845Ssam KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked")); 632178354Ssam 633178354Ssam // The timer LVT always uses the fixed delivery mode. 634178354Ssam lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT); 635178354Ssam if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) { 636178354Ssam VLAPIC_CTR0(vlapic, "vlapic timer fired"); 637148314Ssam vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1); 638184285Ssam } 639184285Ssam} 640148314Ssam 641195618Srpaulostatic VMM_STAT(VLAPIC_INTR_CMC, 642178354Ssam "corrected machine check interrupts generated by vlapic"); 643148314Ssam 644191544Ssamvoid 645184282Ssamvlapic_fire_cmci(struct vlapic *vlapic) 646170530Ssam{ 647170530Ssam uint32_t lvt; 648170530Ssam 649148314Ssam lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT); 650148314Ssam if (vlapic_fire_lvt(vlapic, lvt)) { 651195618Srpaulo vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1); 652222682Sbschmidt } 653191544Ssam} 654191544Ssam 655148314Ssamstatic VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1, 656248106Sadrian "lvts triggered"); 657248069Sadrian 658148314Ssamint 659148314Ssamvlapic_trigger_lvt(struct vlapic *vlapic, int vector) 660178354Ssam{ 661148314Ssam uint32_t lvt; 662148314Ssam 663148314Ssam if (vlapic_enabled(vlapic) == false) { 664148314Ssam /* 665148314Ssam * When the local APIC is global/hardware disabled, 666148314Ssam * LINT[1:0] pins are configured as INTR and NMI pins, 667148314Ssam * respectively. 668148314Ssam */ 669148314Ssam switch (vector) { 670148314Ssam case APIC_LVT_LINT0: 671148314Ssam vm_inject_extint(vlapic->vm, vlapic->vcpuid); 672148314Ssam break; 673148314Ssam case APIC_LVT_LINT1: 674148314Ssam vm_inject_nmi(vlapic->vm, vlapic->vcpuid); 675148314Ssam break; 676148314Ssam default: 677148314Ssam break; 678148314Ssam } 679148314Ssam return (0); 680170530Ssam } 681170530Ssam 682178354Ssam switch (vector) { 683178354Ssam case APIC_LVT_LINT0: 684170530Ssam lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT); 685170530Ssam break; 686170530Ssam case APIC_LVT_LINT1: 687195618Srpaulo lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT); 688195618Srpaulo break; 689195618Srpaulo case APIC_LVT_TIMER: 690195618Srpaulo lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT); 691195618Srpaulo lvt |= APIC_LVT_DM_FIXED; 692195618Srpaulo break; 693195618Srpaulo case APIC_LVT_ERROR: 694195618Srpaulo lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT); 695195618Srpaulo lvt |= APIC_LVT_DM_FIXED; 696195618Srpaulo break; 697195618Srpaulo case APIC_LVT_PMC: 698195618Srpaulo lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT); 699195618Srpaulo break; 700195618Srpaulo case APIC_LVT_THERMAL: 701195618Srpaulo lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT); 702195618Srpaulo break; 703195618Srpaulo case APIC_LVT_CMCI: 704195618Srpaulo lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT); 705148314Ssam break; 706148314Ssam default: 707148314Ssam return (EINVAL); 708148314Ssam } 709148314Ssam if (vlapic_fire_lvt(vlapic, lvt)) { 710148314Ssam vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, 711148314Ssam LVTS_TRIGGERRED, vector, 1); 712195618Srpaulo } 713195618Srpaulo return (0); 714195618Srpaulo} 715195618Srpaulo 716195618Srpaulostatic void 717195618Srpaulovlapic_callout_handler(void *arg) 718148314Ssam{ 719170530Ssam struct vlapic *vlapic; 720191544Ssam struct bintime bt, btnow; 721234324Sadrian sbintime_t rem_sbt; 722222682Sbschmidt 723222682Sbschmidt vlapic = arg; 724222682Sbschmidt 725222682Sbschmidt VLAPIC_TIMER_LOCK(vlapic); 726222682Sbschmidt if (callout_pending(&vlapic->callout)) /* callout was reset */ 727222682Sbschmidt goto done; 728222682Sbschmidt 729222682Sbschmidt if (!callout_active(&vlapic->callout)) /* callout was stopped */ 730191544Ssam goto done; 731191544Ssam 732191544Ssam callout_deactivate(&vlapic->callout); 733148314Ssam 734148314Ssam vlapic_fire_timer(vlapic); 735148314Ssam 736148314Ssam if (vlapic_periodic_timer(vlapic)) { 737119150Ssam binuptime(&btnow); 738119150Ssam KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=), 739119150Ssam ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx", 740119150Ssam btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec, 741178354Ssam vlapic->timer_fire_bt.frac)); 742178354Ssam 743119150Ssam /* 744170530Ssam * Compute the delta between when the timer was supposed to 745184282Ssam * fire and the present time. 746184282Ssam */ 747116742Ssam bt = btnow; 748178354Ssam bintime_sub(&bt, &vlapic->timer_fire_bt); 749178354Ssam 750116742Ssam rem_sbt = bttosbt(vlapic->timer_period_bt); 751248069Sadrian if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) { 752116742Ssam /* 753119150Ssam * Adjust the time until the next countdown downward 754116742Ssam * to account for the lost time. 755178354Ssam */ 756178354Ssam rem_sbt -= bttosbt(bt); 757178354Ssam } else { 758178354Ssam /* 759178354Ssam * If the delta is greater than the timer period then 760178354Ssam * just reset our time base instead of trying to catch 761178354Ssam * up. 762178354Ssam */ 763178354Ssam vlapic->timer_fire_bt = btnow; 764178354Ssam VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu " 765178354Ssam "usecs, period is %lu usecs - resetting time base", 766178354Ssam bttosbt(bt) / SBT_1US, 767243882Sglebius bttosbt(vlapic->timer_period_bt) / SBT_1US); 768178354Ssam } 769178354Ssam 770116742Ssam bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt); 771178354Ssam callout_reset_sbt(&vlapic->callout, rem_sbt, 0, 772119150Ssam vlapic_callout_handler, vlapic, 0); 773248069Sadrian } 774248069Sadriandone: 775116742Ssam VLAPIC_TIMER_UNLOCK(vlapic); 776191544Ssam} 777184282Ssam 778184282Ssamvoid 779184282Ssamvlapic_icrtmr_write_handler(struct vlapic *vlapic) 780178354Ssam{ 781178354Ssam struct LAPIC *lapic; 782262007Skevlo sbintime_t sbt; 783138568Ssam uint32_t icr_timer; 784184286Ssam 785184282Ssam VLAPIC_TIMER_LOCK(vlapic); 786184282Ssam 787184282Ssam lapic = vlapic->apic_page; 788184282Ssam icr_timer = lapic->icr_timer; 789116742Ssam 790138568Ssam vlapic->timer_period_bt = vlapic->timer_freq_bt; 791178354Ssam bintime_mul(&vlapic->timer_period_bt, icr_timer); 792178354Ssam 793138568Ssam if (icr_timer != 0) { 794138568Ssam binuptime(&vlapic->timer_fire_bt); 795138568Ssam bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt); 796138568Ssam 797138568Ssam sbt = bttosbt(vlapic->timer_period_bt); 798148936Ssam callout_reset_sbt(&vlapic->callout, sbt, 0, 799138568Ssam vlapic_callout_handler, vlapic, 0); 800116742Ssam } else 801138568Ssam callout_stop(&vlapic->callout); 802170530Ssam 803248069Sadrian VLAPIC_TIMER_UNLOCK(vlapic); 804248069Sadrian} 805248069Sadrian 806116742Ssam/* 807116742Ssam * This function populates 'dmask' with the set of vcpus that match the 808119150Ssam * addressing specified by the (dest, phys, lowprio) tuple. 809184283Ssam * 810184283Ssam * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit) 811184283Ssam * or xAPIC (8-bit) destination field. 812148582Ssam */ 813148582Ssamstatic void 814148582Ssamvlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys, 815178354Ssam bool lowprio, bool x2apic_dest) 816178354Ssam{ 817178354Ssam struct vlapic *vlapic; 818178354Ssam uint32_t dfr, ldr, ldest, cluster; 819119150Ssam uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id; 820138568Ssam cpuset_t amask; 821148301Ssam int vcpuid; 822138568Ssam 823178354Ssam if ((x2apic_dest && dest == 0xffffffff) || 824148301Ssam (!x2apic_dest && dest == 0xff)) { 825138568Ssam /* 826138568Ssam * Broadcast in both logical and physical modes. 827184283Ssam */ 828184283Ssam *dmask = vm_active_cpus(vm); 829248069Sadrian return; 830138568Ssam } 831178354Ssam 832178354Ssam if (phys) { 833178354Ssam /* 834178354Ssam * Physical mode: destination is APIC ID. 835178354Ssam */ 836178354Ssam CPU_ZERO(dmask); 837178354Ssam vcpuid = vm_apicid2vcpuid(vm, dest); 838178354Ssam if (vcpuid < VM_MAXCPU) 839184283Ssam CPU_SET(vcpuid, dmask); 840184283Ssam } else { 841184283Ssam /* 842184283Ssam * In the "Flat Model" the MDA is interpreted as an 8-bit wide 843184283Ssam * bitmask. This model is only avilable in the xAPIC mode. 844184283Ssam */ 845184283Ssam mda_flat_ldest = dest & 0xff; 846184283Ssam 847184283Ssam /* 848184283Ssam * In the "Cluster Model" the MDA is used to identify a 849184283Ssam * specific cluster and a set of APICs in that cluster. 850138568Ssam */ 851138568Ssam if (x2apic_dest) { 852170530Ssam mda_cluster_id = dest >> 16; 853178354Ssam mda_cluster_ldest = dest & 0xffff; 854138568Ssam } else { 855138568Ssam mda_cluster_id = (dest >> 4) & 0xf; 856184283Ssam mda_cluster_ldest = dest & 0xf; 857184283Ssam } 858243882Sglebius 859184283Ssam /* 860184283Ssam * Logical mode: match each APIC that has a bit set 861184283Ssam * in it's LDR that matches a bit in the ldest. 862184283Ssam */ 863184283Ssam CPU_ZERO(dmask); 864138568Ssam amask = vm_active_cpus(vm); 865248069Sadrian while ((vcpuid = CPU_FFS(&amask)) != 0) { 866248069Sadrian vcpuid--; 867184283Ssam CPU_CLR(vcpuid, &amask); 868184283Ssam 869184283Ssam vlapic = vm_lapic(vm, vcpuid); 870184283Ssam dfr = vlapic->apic_page->dfr; 871184283Ssam ldr = vlapic->apic_page->ldr; 872191544Ssam 873184283Ssam if ((dfr & APIC_DFR_MODEL_MASK) == 874184283Ssam APIC_DFR_MODEL_FLAT) { 875184283Ssam ldest = ldr >> 24; 876184283Ssam mda_ldest = mda_flat_ldest; 877184283Ssam } else if ((dfr & APIC_DFR_MODEL_MASK) == 878184283Ssam APIC_DFR_MODEL_CLUSTER) { 879184283Ssam if (x2apic(vlapic)) { 880184283Ssam cluster = ldr >> 16; 881184283Ssam ldest = ldr & 0xffff; 882184283Ssam } else { 883184283Ssam cluster = ldr >> 28; 884184283Ssam ldest = (ldr >> 24) & 0xf; 885191544Ssam } 886184283Ssam if (cluster != mda_cluster_id) 887184283Ssam continue; 888184283Ssam mda_ldest = mda_cluster_ldest; 889184283Ssam } else { 890178354Ssam /* 891178354Ssam * Guest has configured a bad logical 892178354Ssam * model for this vcpu - skip it. 893178354Ssam */ 894178354Ssam VLAPIC_CTR1(vlapic, "vlapic has bad logical " 895178354Ssam "model %x - cannot deliver interrupt", dfr); 896184283Ssam continue; 897184286Ssam } 898184283Ssam 899172231Ssam if ((mda_ldest & ldest) != 0) { 900138568Ssam CPU_SET(vcpuid, dmask); 901138568Ssam if (lowprio) 902138568Ssam break; 903178354Ssam } 904184283Ssam } 905184283Ssam } 906148936Ssam} 907148314Ssam 908148314Ssamstatic VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu"); 909248069Sadrian 910248069Sadrianstatic void 911248069Sadrianvlapic_set_tpr(struct vlapic *vlapic, uint8_t val) 912138568Ssam{ 913138568Ssam struct LAPIC *lapic = vlapic->apic_page; 914138568Ssam 915138568Ssam if (lapic->tpr != val) { 916138568Ssam VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vlapic TPR changed " 917138568Ssam "from %#x to %#x", lapic->tpr, val); 918138568Ssam lapic->tpr = val; 919138568Ssam vlapic_update_ppr(vlapic); 920138568Ssam } 921178354Ssam} 922138568Ssam 923178354Ssamstatic uint8_t 924138568Ssamvlapic_get_tpr(struct vlapic *vlapic) 925138568Ssam{ 926178354Ssam struct LAPIC *lapic = vlapic->apic_page; 927178354Ssam 928178354Ssam return (lapic->tpr); 929178354Ssam} 930178354Ssam 931178354Ssamvoid 932178354Ssamvlapic_set_cr8(struct vlapic *vlapic, uint64_t val) 933178354Ssam{ 934178354Ssam uint8_t tpr; 935178354Ssam 936178354Ssam if (val & ~0xf) { 937178354Ssam vm_inject_gp(vlapic->vm, vlapic->vcpuid); 938138568Ssam return; 939138568Ssam } 940138568Ssam 941138568Ssam tpr = val << 4; 942138568Ssam vlapic_set_tpr(vlapic, tpr); 943138568Ssam} 944138568Ssam 945138568Ssamuint64_t 946138568Ssamvlapic_get_cr8(struct vlapic *vlapic) 947138568Ssam{ 948138568Ssam uint8_t tpr; 949162375Sandre 950138568Ssam tpr = vlapic_get_tpr(vlapic); 951138568Ssam return (tpr >> 4); 952138568Ssam} 953162375Sandre 954138568Ssamint 955138568Ssamvlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu) 956138568Ssam{ 957138568Ssam int i; 958138568Ssam bool phys; 959173867Ssam cpuset_t dmask; 960138568Ssam uint64_t icrval; 961138568Ssam uint32_t dest, vec, mode; 962193658Ssam struct vlapic *vlapic2; 963138568Ssam struct vm_exit *vmexit; 964138568Ssam struct LAPIC *lapic; 965173867Ssam 966138568Ssam lapic = vlapic->apic_page; 967173867Ssam lapic->icr_lo &= ~APIC_DELSTAT_PEND; 968138568Ssam icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo; 969173867Ssam 970173867Ssam if (x2apic(vlapic)) 971173867Ssam dest = icrval >> 32; 972173867Ssam else 973173867Ssam dest = icrval >> (32 + 24); 974138568Ssam vec = icrval & APIC_VECTOR_MASK; 975138568Ssam mode = icrval & APIC_DELMODE_MASK; 976193658Ssam 977193658Ssam if (mode == APIC_DELMODE_FIXED && vec < 16) { 978193658Ssam vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR); 979193658Ssam VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec); 980193658Ssam return (0); 981229795Sbz } 982193658Ssam 983193658Ssam VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec); 984193658Ssam 985193658Ssam if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) { 986193658Ssam switch (icrval & APIC_DEST_MASK) { 987193658Ssam case APIC_DEST_DESTFLD: 988193658Ssam phys = ((icrval & APIC_DESTMODE_LOG) == 0); 989193658Ssam vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false, 990193658Ssam x2apic(vlapic)); 991138568Ssam break; 992193658Ssam case APIC_DEST_SELF: 993193658Ssam CPU_SETOF(vlapic->vcpuid, &dmask); 994193658Ssam break; 995138568Ssam case APIC_DEST_ALLISELF: 996138568Ssam dmask = vm_active_cpus(vlapic->vm); 997138568Ssam break; 998138568Ssam case APIC_DEST_ALLESELF: 999138568Ssam dmask = vm_active_cpus(vlapic->vm); 1000138568Ssam CPU_CLR(vlapic->vcpuid, &dmask); 1001138568Ssam break; 1002138568Ssam default: 1003138568Ssam CPU_ZERO(&dmask); /* satisfy gcc */ 1004138568Ssam break; 1005138568Ssam } 1006138568Ssam 1007138568Ssam while ((i = CPU_FFS(&dmask)) != 0) { 1008138568Ssam i--; 1009178354Ssam CPU_CLR(i, &dmask); 1010138568Ssam if (mode == APIC_DELMODE_FIXED) { 1011138568Ssam lapic_intr_edge(vlapic->vm, i, vec); 1012138568Ssam vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, 1013138568Ssam IPIS_SENT, i, 1); 1014138568Ssam VLAPIC_CTR2(vlapic, "vlapic sending ipi %d " 1015138568Ssam "to vcpuid %d", vec, i); 1016178354Ssam } else { 1017178354Ssam vm_inject_nmi(vlapic->vm, i); 1018138568Ssam VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi " 1019138568Ssam "to vcpuid %d", i); 1020138568Ssam } 1021138568Ssam } 1022138568Ssam 1023138568Ssam return (0); /* handled completely in the kernel */ 1024138568Ssam } 1025138568Ssam 1026138568Ssam if (mode == APIC_DELMODE_INIT) { 1027138568Ssam if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT) 1028139527Ssam return (0); 1029139527Ssam 1030139527Ssam if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) { 1031139527Ssam vlapic2 = vm_lapic(vlapic->vm, dest); 1032139527Ssam 1033190391Ssam /* move from INIT to waiting-for-SIPI state */ 1034178354Ssam if (vlapic2->boot_state == BS_INIT) { 1035139527Ssam vlapic2->boot_state = BS_SIPI; 1036139527Ssam } 1037164805Ssam 1038178354Ssam return (0); 1039139527Ssam } 1040139527Ssam } 1041139527Ssam 1042139527Ssam if (mode == APIC_DELMODE_STARTUP) { 1043139527Ssam if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) { 1044156758Ssam vlapic2 = vm_lapic(vlapic->vm, dest); 1045156758Ssam 1046156758Ssam /* 1047156758Ssam * Ignore SIPIs in any state other than wait-for-SIPI 1048156758Ssam */ 1049156758Ssam if (vlapic2->boot_state != BS_SIPI) 1050179394Ssam return (0); 1051156758Ssam 1052156758Ssam vlapic2->boot_state = BS_RUNNING; 1053178354Ssam 1054156758Ssam *retu = true; 1055178354Ssam vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid); 1056156758Ssam vmexit->exitcode = VM_EXITCODE_SPINUP_AP; 1057156758Ssam vmexit->u.spinup_ap.vcpu = dest; 1058156758Ssam vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT; 1059139527Ssam 1060139527Ssam return (0); 1061139527Ssam } 1062139527Ssam } 1063139527Ssam 1064164805Ssam /* 1065139527Ssam * This will cause a return to userland. 1066139527Ssam */ 1067139527Ssam return (1); 1068139527Ssam} 1069139527Ssam 1070139527Ssamvoid 1071139527Ssamvlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val) 1072178354Ssam{ 1073139527Ssam int vec; 1074178354Ssam 1075139527Ssam KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode")); 1076139527Ssam 1077139527Ssam vec = val & 0xff; 1078139527Ssam lapic_intr_edge(vlapic->vm, vlapic->vcpuid, vec); 1079253007Salfred vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, IPIS_SENT, 1080139527Ssam vlapic->vcpuid, 1); 1081139527Ssam VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec); 1082139527Ssam} 1083139527Ssam 1084139527Ssamint 1085139527Ssamvlapic_pending_intr(struct vlapic *vlapic, int *vecptr) 1086139527Ssam{ 1087139527Ssam struct LAPIC *lapic = vlapic->apic_page; 1088139527Ssam int idx, i, bitpos, vector; 1089139527Ssam uint32_t *irrptr, val; 1090139527Ssam 1091139527Ssam if (vlapic->ops.pending_intr) 1092139527Ssam return ((*vlapic->ops.pending_intr)(vlapic, vecptr)); 1093139527Ssam 1094139527Ssam irrptr = &lapic->irr0; 1095139527Ssam 1096139527Ssam /* 1097139527Ssam * The x86 architecture reserves the the first 32 vectors for use 1098139527Ssam * by the processor. 1099139527Ssam */ 1100139527Ssam for (i = 7; i > 0; i--) { 1101139527Ssam idx = i * 4; 1102139527Ssam val = atomic_load_acq_int(&irrptr[idx]); 1103139527Ssam bitpos = fls(val); 1104139527Ssam if (bitpos != 0) { 1105139527Ssam vector = i * 32 + (bitpos - 1); 1106139527Ssam if (PRIO(vector) > PRIO(lapic->ppr)) { 1107139527Ssam VLAPIC_CTR1(vlapic, "pending intr %d", vector); 1108139527Ssam if (vecptr != NULL) 1109139527Ssam *vecptr = vector; 1110139527Ssam return (1); 1111139527Ssam } else 1112139527Ssam break; 1113139527Ssam } 1114138568Ssam } 1115138568Ssam return (0); 1116178354Ssam} 1117178354Ssam 1118138568Ssamvoid 1119167432Ssamvlapic_intr_accepted(struct vlapic *vlapic, int vector) 1120178354Ssam{ 1121178354Ssam struct LAPIC *lapic = vlapic->apic_page; 1122138568Ssam uint32_t *irrptr, *isrptr; 1123178354Ssam int idx, stk_top; 1124138568Ssam 1125138568Ssam if (vlapic->ops.intr_accepted) 1126138568Ssam return ((*vlapic->ops.intr_accepted)(vlapic, vector)); 1127138568Ssam 1128138568Ssam /* 1129138568Ssam * clear the ready bit for vector being accepted in irr 1130139527Ssam * and set the vector as in service in isr. 1131139527Ssam */ 1132139527Ssam idx = (vector / 32) * 4; 1133139527Ssam 1134139527Ssam irrptr = &lapic->irr0; 1135178354Ssam atomic_clear_int(&irrptr[idx], 1 << (vector % 32)); 1136178354Ssam VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted"); 1137139527Ssam 1138178354Ssam isrptr = &lapic->isr0; 1139178354Ssam isrptr[idx] |= 1 << (vector % 32); 1140139527Ssam VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted"); 1141178354Ssam 1142139527Ssam /* 1143139527Ssam * Update the PPR 1144139527Ssam */ 1145138568Ssam vlapic->isrvec_stk_top++; 1146138568Ssam 1147138568Ssam stk_top = vlapic->isrvec_stk_top; 1148138568Ssam if (stk_top >= ISRVEC_STK_SIZE) 1149178354Ssam panic("isrvec_stk_top overflow %d", stk_top); 1150178354Ssam 1151178354Ssam vlapic->isrvec_stk[stk_top] = vector; 1152138568Ssam vlapic_update_ppr(vlapic); 1153116742Ssam} 1154190579Ssam 1155190579Ssamvoid 1156116742Ssamvlapic_svr_write_handler(struct vlapic *vlapic) 1157178354Ssam{ 1158234878Smonthadar struct LAPIC *lapic; 1159178354Ssam uint32_t old, new, changed; 1160195618Srpaulo 1161195618Srpaulo lapic = vlapic->apic_page; 1162195784Srpaulo 1163234878Smonthadar new = lapic->svr; 1164234878Smonthadar old = vlapic->svr_last; 1165195618Srpaulo vlapic->svr_last = new; 1166116742Ssam 1167116742Ssam changed = old ^ new; 1168138568Ssam if ((changed & APIC_SVR_ENABLE) != 0) { 1169116742Ssam if ((new & APIC_SVR_ENABLE) == 0) { 1170190391Ssam /* 1171191544Ssam * The apic is now disabled so stop the apic timer 1172195618Srpaulo * and mask all the LVT entries. 1173195618Srpaulo */ 1174248069Sadrian VLAPIC_CTR0(vlapic, "vlapic is software-disabled"); 1175248069Sadrian VLAPIC_TIMER_LOCK(vlapic); 1176116742Ssam callout_stop(&vlapic->callout); 1177170530Ssam VLAPIC_TIMER_UNLOCK(vlapic); 1178170530Ssam vlapic_mask_lvts(vlapic); 1179170530Ssam } else { 1180170530Ssam /* 1181170530Ssam * The apic is now enabled so restart the apic timer 1182138568Ssam * if it is configured in periodic mode. 1183178354Ssam */ 1184116742Ssam VLAPIC_CTR0(vlapic, "vlapic is software-enabled"); 1185138568Ssam if (vlapic_periodic_timer(vlapic)) 1186138568Ssam vlapic_icrtmr_write_handler(vlapic); 1187138568Ssam } 1188138568Ssam } 1189138568Ssam} 1190138568Ssam 1191138568Ssamint 1192138568Ssamvlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset, 1193138568Ssam uint64_t *data, bool *retu) 1194138568Ssam{ 1195138568Ssam struct LAPIC *lapic = vlapic->apic_page; 1196138568Ssam uint32_t *reg; 1197178354Ssam int i; 1198178354Ssam 1199178354Ssam /* Ignore MMIO accesses in x2APIC mode */ 1200178354Ssam if (x2apic(vlapic) && mmio_access) { 1201178354Ssam VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode", 1202178354Ssam offset); 1203139527Ssam *data = 0; 1204178354Ssam goto done; 1205178354Ssam } 1206178354Ssam 1207178354Ssam if (!x2apic(vlapic) && !mmio_access) { 1208178354Ssam /* 1209178354Ssam * XXX Generate GP fault for MSR accesses in xAPIC mode 1210178354Ssam */ 1211171950Ssam VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in " 1212138568Ssam "xAPIC mode", offset); 1213138568Ssam *data = 0; 1214138568Ssam goto done; 1215139527Ssam } 1216139527Ssam 1217139527Ssam if (offset > sizeof(*lapic)) { 1218139527Ssam *data = 0; 1219139527Ssam goto done; 1220139527Ssam } 1221232480Sadrian 1222139527Ssam offset &= ~3; 1223232480Sadrian switch(offset) 1224232480Sadrian { 1225232480Sadrian case APIC_OFFSET_ID: 1226139527Ssam *data = lapic->id; 1227138568Ssam break; 1228138568Ssam case APIC_OFFSET_VER: 1229138568Ssam *data = lapic->version; 1230195618Srpaulo break; 1231195618Srpaulo case APIC_OFFSET_TPR: 1232195618Srpaulo *data = vlapic_get_tpr(vlapic); 1233195618Srpaulo break; 1234195618Srpaulo case APIC_OFFSET_APR: 1235195618Srpaulo *data = lapic->apr; 1236195618Srpaulo break; 1237195618Srpaulo case APIC_OFFSET_PPR: 1238195618Srpaulo *data = lapic->ppr; 1239195618Srpaulo break; 1240195618Srpaulo case APIC_OFFSET_EOI: 1241195618Srpaulo *data = lapic->eoi; 1242195618Srpaulo break; 1243195618Srpaulo case APIC_OFFSET_LDR: 1244195618Srpaulo *data = lapic->ldr; 1245195618Srpaulo break; 1246195618Srpaulo case APIC_OFFSET_DFR: 1247234878Smonthadar *data = lapic->dfr; 1248234878Smonthadar break; 1249234878Smonthadar case APIC_OFFSET_SVR: 1250234878Smonthadar *data = lapic->svr; 1251234878Smonthadar break; 1252234878Smonthadar case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 1253234878Smonthadar i = (offset - APIC_OFFSET_ISR0) >> 2; 1254234878Smonthadar reg = &lapic->isr0; 1255234878Smonthadar *data = *(reg + i); 1256234878Smonthadar break; 1257234878Smonthadar case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 1258234878Smonthadar i = (offset - APIC_OFFSET_TMR0) >> 2; 1259234878Smonthadar reg = &lapic->tmr0; 1260234878Smonthadar *data = *(reg + i); 1261234878Smonthadar break; 1262234878Smonthadar case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 1263234878Smonthadar i = (offset - APIC_OFFSET_IRR0) >> 2; 1264234878Smonthadar reg = &lapic->irr0; 1265234878Smonthadar *data = atomic_load_acq_int(reg + i); 1266234878Smonthadar break; 1267234878Smonthadar case APIC_OFFSET_ESR: 1268234878Smonthadar *data = lapic->esr; 1269195618Srpaulo break; 1270234878Smonthadar case APIC_OFFSET_ICR_LOW: 1271234878Smonthadar *data = lapic->icr_lo; 1272234878Smonthadar if (x2apic(vlapic)) 1273234878Smonthadar *data |= (uint64_t)lapic->icr_hi << 32; 1274234878Smonthadar break; 1275234878Smonthadar case APIC_OFFSET_ICR_HI: 1276234878Smonthadar *data = lapic->icr_hi; 1277234878Smonthadar break; 1278234878Smonthadar case APIC_OFFSET_CMCI_LVT: 1279234878Smonthadar case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 1280234878Smonthadar *data = vlapic_get_lvt(vlapic, offset); 1281195618Srpaulo#ifdef INVARIANTS 1282195618Srpaulo reg = vlapic_get_lvtptr(vlapic, offset); 1283195618Srpaulo KASSERT(*data == *reg, ("inconsistent lvt value at " 1284195618Srpaulo "offset %#lx: %#lx/%#x", offset, *data, *reg)); 1285195618Srpaulo#endif 1286195618Srpaulo break; 1287195618Srpaulo case APIC_OFFSET_TIMER_ICR: 1288195618Srpaulo *data = lapic->icr_timer; 1289195618Srpaulo break; 1290195618Srpaulo case APIC_OFFSET_TIMER_CCR: 1291195618Srpaulo *data = vlapic_get_ccr(vlapic); 1292195618Srpaulo break; 1293195618Srpaulo case APIC_OFFSET_TIMER_DCR: 1294195618Srpaulo *data = lapic->dcr_timer; 1295195618Srpaulo break; 1296195618Srpaulo case APIC_OFFSET_SELF_IPI: 1297195618Srpaulo /* 1298195618Srpaulo * XXX generate a GP fault if vlapic is in x2apic mode 1299178354Ssam */ 1300178354Ssam *data = 0; 1301178354Ssam break; 1302138568Ssam case APIC_OFFSET_RRR: 1303178354Ssam default: 1304178354Ssam *data = 0; 1305178354Ssam break; 1306170530Ssam } 1307190391Ssamdone: 1308170530Ssam VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data); 1309170530Ssam return 0; 1310170530Ssam} 1311195618Srpaulo 1312170530Ssamint 1313170530Ssamvlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset, 1314170530Ssam uint64_t data, bool *retu) 1315170530Ssam{ 1316170530Ssam struct LAPIC *lapic = vlapic->apic_page; 1317170530Ssam uint32_t *regptr; 1318170530Ssam int retval; 1319170530Ssam 1320170530Ssam KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE, 1321170530Ssam ("vlapic_write: invalid offset %#lx", offset)); 1322170530Ssam 1323170530Ssam VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx", 1324170530Ssam offset, data); 1325190391Ssam 1326190391Ssam if (offset > sizeof(*lapic)) 1327190579Ssam return (0); 1328190579Ssam 1329190579Ssam /* Ignore MMIO accesses in x2APIC mode */ 1330195618Srpaulo if (x2apic(vlapic) && mmio_access) { 1331190391Ssam VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx " 1332190391Ssam "in x2APIC mode", data, offset); 1333190391Ssam return (0); 1334127772Ssam } 1335138568Ssam 1336138568Ssam /* 1337243882Sglebius * XXX Generate GP fault for MSR accesses in xAPIC mode 1338121180Ssam */ 1339178354Ssam if (!x2apic(vlapic) && !mmio_access) { 1340119150Ssam VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx " 1341121180Ssam "in xAPIC mode", data, offset); 1342116742Ssam return (0); 1343116742Ssam } 1344170530Ssam 1345195618Srpaulo retval = 0; 1346178354Ssam switch(offset) 1347178354Ssam { 1348178354Ssam case APIC_OFFSET_ID: 1349178354Ssam lapic->id = data; 1350178354Ssam vlapic_id_write_handler(vlapic); 1351178354Ssam break; 1352178354Ssam case APIC_OFFSET_TPR: 1353116742Ssam vlapic_set_tpr(vlapic, data & 0xff); 1354116742Ssam break; 1355116742Ssam case APIC_OFFSET_EOI: 1356116742Ssam vlapic_process_eoi(vlapic); 1357116742Ssam break; 1358116742Ssam case APIC_OFFSET_LDR: 1359116742Ssam lapic->ldr = data; 1360116742Ssam vlapic_ldr_write_handler(vlapic); 1361116742Ssam break; 1362116742Ssam case APIC_OFFSET_DFR: 1363116742Ssam lapic->dfr = data; 1364140636Ssam vlapic_dfr_write_handler(vlapic); 1365178354Ssam break; 1366140636Ssam case APIC_OFFSET_SVR: 1367140636Ssam lapic->svr = data; 1368178354Ssam vlapic_svr_write_handler(vlapic); 1369116742Ssam break; 1370116742Ssam case APIC_OFFSET_ICR_LOW: 1371116742Ssam lapic->icr_lo = data; 1372116742Ssam if (x2apic(vlapic)) 1373116742Ssam lapic->icr_hi = data >> 32; 1374116742Ssam retval = vlapic_icrlo_write_handler(vlapic, retu); 1375116742Ssam break; 1376195618Srpaulo case APIC_OFFSET_ICR_HI: 1377195618Srpaulo lapic->icr_hi = data; 1378195618Srpaulo break; 1379195784Srpaulo case APIC_OFFSET_CMCI_LVT: 1380195618Srpaulo case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 1381234878Smonthadar regptr = vlapic_get_lvtptr(vlapic, offset); 1382195618Srpaulo *regptr = data; 1383234878Smonthadar vlapic_lvt_write_handler(vlapic, offset); 1384195618Srpaulo break; 1385234878Smonthadar case APIC_OFFSET_TIMER_ICR: 1386234878Smonthadar lapic->icr_timer = data; 1387234878Smonthadar vlapic_icrtmr_write_handler(vlapic); 1388234878Smonthadar break; 1389234878Smonthadar 1390234878Smonthadar case APIC_OFFSET_TIMER_DCR: 1391234878Smonthadar lapic->dcr_timer = data; 1392234878Smonthadar vlapic_dcr_write_handler(vlapic); 1393234878Smonthadar break; 1394234878Smonthadar 1395234878Smonthadar case APIC_OFFSET_ESR: 1396234878Smonthadar vlapic_esr_write_handler(vlapic); 1397234878Smonthadar break; 1398234878Smonthadar 1399234878Smonthadar case APIC_OFFSET_SELF_IPI: 1400234878Smonthadar if (x2apic(vlapic)) 1401234878Smonthadar vlapic_self_ipi_handler(vlapic, data); 1402234878Smonthadar break; 1403234878Smonthadar 1404234878Smonthadar case APIC_OFFSET_VER: 1405234878Smonthadar case APIC_OFFSET_APR: 1406234878Smonthadar case APIC_OFFSET_PPR: 1407195618Srpaulo case APIC_OFFSET_RRR: 1408234878Smonthadar case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 1409195618Srpaulo case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 1410195618Srpaulo case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 1411195618Srpaulo case APIC_OFFSET_TIMER_CCR: 1412195784Srpaulo default: 1413195618Srpaulo // Read only. 1414234878Smonthadar break; 1415234878Smonthadar } 1416195618Srpaulo 1417195618Srpaulo return (retval); 1418234878Smonthadar} 1419234878Smonthadar 1420234878Smonthadarstatic void 1421195618Srpaulovlapic_reset(struct vlapic *vlapic) 1422234878Smonthadar{ 1423195784Srpaulo struct LAPIC *lapic; 1424234878Smonthadar 1425234878Smonthadar lapic = vlapic->apic_page; 1426234878Smonthadar bzero(lapic, sizeof(struct LAPIC)); 1427195618Srpaulo 1428195618Srpaulo lapic->id = vlapic_get_id(vlapic); 1429195618Srpaulo lapic->version = VLAPIC_VERSION; 1430195618Srpaulo lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT); 1431195618Srpaulo lapic->dfr = 0xffffffff; 1432195618Srpaulo lapic->svr = APIC_SVR_VECTOR; 1433195618Srpaulo vlapic_mask_lvts(vlapic); 1434195618Srpaulo vlapic_reset_tmr(vlapic); 1435195618Srpaulo 1436195618Srpaulo lapic->dcr_timer = 0; 1437195618Srpaulo vlapic_dcr_write_handler(vlapic); 1438178354Ssam 1439195618Srpaulo if (vlapic->vcpuid == 0) 1440119150Ssam vlapic->boot_state = BS_RUNNING; /* BSP */ 1441116742Ssam else 1442190678Ssam vlapic->boot_state = BS_INIT; /* AP */ 1443147789Ssam 1444139527Ssam vlapic->svr_last = lapic->svr; 1445138568Ssam} 1446138568Ssam 1447178354Ssamvoid 1448178354Ssamvlapic_init(struct vlapic *vlapic) 1449195618Srpaulo{ 1450195618Srpaulo KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized")); 1451178354Ssam KASSERT(vlapic->vcpuid >= 0 && vlapic->vcpuid < VM_MAXCPU, 1452138568Ssam ("vlapic_init: vcpuid is not initialized")); 1453138568Ssam KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not " 1454138568Ssam "initialized")); 1455178354Ssam 1456138568Ssam /* 1457178354Ssam * If the vlapic is configured in x2apic mode then it will be 1458232480Sadrian * accessed in the critical section via the MSR emulation code. 1459237561Smonthadar * 1460237561Smonthadar * Therefore the timer mutex must be a spinlock because blockable 1461237561Smonthadar * mutexes cannot be acquired in a critical section. 1462232480Sadrian */ 1463232480Sadrian mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN); 1464178354Ssam callout_init(&vlapic->callout, 1); 1465138568Ssam 1466183247Ssam vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED; 1467183247Ssam 1468183247Ssam if (vlapic->vcpuid == 0) 1469183247Ssam vlapic->msr_apicbase |= APICBASE_BSP; 1470183247Ssam 1471183247Ssam vlapic_reset(vlapic); 1472183247Ssam} 1473183247Ssam 1474183247Ssamvoid 1475183247Ssamvlapic_cleanup(struct vlapic *vlapic) 1476183247Ssam{ 1477183247Ssam 1478183247Ssam callout_drain(&vlapic->callout); 1479191544Ssam} 1480183247Ssam 1481191544Ssamuint64_t 1482191571Ssamvlapic_get_apicbase(struct vlapic *vlapic) 1483183247Ssam{ 1484138568Ssam 1485191544Ssam return (vlapic->msr_apicbase); 1486170530Ssam} 1487191544Ssam 1488191571Ssamint 1489138568Ssamvlapic_set_apicbase(struct vlapic *vlapic, uint64_t new) 1490191571Ssam{ 1491195618Srpaulo 1492170530Ssam if (vlapic->msr_apicbase != new) { 1493178354Ssam VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx " 1494170530Ssam "not supported", vlapic->msr_apicbase, new); 1495178354Ssam return (-1); 1496191545Ssam } 1497139527Ssam 1498139527Ssam return (0); 1499139527Ssam} 1500139527Ssam 1501139527Ssamvoid 1502178354Ssamvlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 1503178354Ssam{ 1504178354Ssam struct vlapic *vlapic; 1505167432Ssam struct LAPIC *lapic; 1506167432Ssam 1507262007Skevlo vlapic = vm_lapic(vm, vcpuid); 1508178354Ssam 1509178354Ssam if (state == X2APIC_DISABLED) 1510178354Ssam vlapic->msr_apicbase &= ~APICBASE_X2APIC; 1511178354Ssam else 1512178354Ssam vlapic->msr_apicbase |= APICBASE_X2APIC; 1513139527Ssam 1514139527Ssam /* 1515139527Ssam * Reset the local APIC registers whose values are mode-dependent. 1516139527Ssam * 1517178354Ssam * XXX this works because the APIC mode can be changed only at vcpu 1518178354Ssam * initialization time. 1519170530Ssam */ 1520138568Ssam lapic = vlapic->apic_page; 1521184286Ssam lapic->id = vlapic_get_id(vlapic); 1522184286Ssam if (x2apic(vlapic)) { 1523138568Ssam lapic->ldr = x2apic_ldr(vlapic); 1524191544Ssam lapic->dfr = 0; 1525161144Ssam } else { 1526191544Ssam lapic->ldr = 0; 1527191544Ssam lapic->dfr = 0xffffffff; 1528161144Ssam } 1529138568Ssam 1530138568Ssam if (state == X2APIC_ENABLED) { 1531116742Ssam if (vlapic->ops.enable_x2apic_mode) 1532119150Ssam (*vlapic->ops.enable_x2apic_mode)(vlapic); 1533119150Ssam } 1534119150Ssam} 1535119150Ssam 1536178354Ssamvoid 1537234878Smonthadarvlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys, 1538116742Ssam int delmode, int vec) 1539116742Ssam{ 1540116742Ssam bool lowprio; 1541170530Ssam int vcpuid; 1542170530Ssam cpuset_t dmask; 1543170530Ssam 1544170530Ssam if (delmode != IOART_DELFIXED && 1545170530Ssam delmode != IOART_DELLOPRI && 1546170530Ssam delmode != IOART_DELEXINT) { 1547170530Ssam VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode); 1548170530Ssam return; 1549178354Ssam } 1550170530Ssam lowprio = (delmode == IOART_DELLOPRI); 1551170530Ssam 1552250974Sadrian /* 1553170530Ssam * We don't provide any virtual interrupt redirection hardware so 1554170530Ssam * all interrupts originating from the ioapic or MSI specify the 1555170530Ssam * 'dest' in the legacy xAPIC format. 1556250974Sadrian */ 1557170530Ssam vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false); 1558170530Ssam 1559170530Ssam while ((vcpuid = CPU_FFS(&dmask)) != 0) { 1560170530Ssam vcpuid--; 1561170530Ssam CPU_CLR(vcpuid, &dmask); 1562250974Sadrian if (delmode == IOART_DELEXINT) { 1563250974Sadrian vm_inject_extint(vm, vcpuid); 1564250974Sadrian } else { 1565250974Sadrian lapic_set_intr(vm, vcpuid, vec, level); 1566250974Sadrian } 1567250974Sadrian } 1568250974Sadrian} 1569250974Sadrian 1570170530Ssamvoid 1571170530Ssamvlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum) 1572170530Ssam{ 1573250974Sadrian /* 1574170530Ssam * Post an interrupt to the vcpu currently running on 'hostcpu'. 1575170530Ssam * 1576170530Ssam * This is done by leveraging features like Posted Interrupts (Intel) 1577170530Ssam * Doorbell MSR (AMD AVIC) that avoid a VM exit. 1578170530Ssam * 1579170530Ssam * If neither of these features are available then fallback to 1580170530Ssam * sending an IPI to 'hostcpu'. 1581170530Ssam */ 1582178354Ssam if (vlapic->ops.post_intr) 1583170530Ssam (*vlapic->ops.post_intr)(vlapic, hostcpu); 1584170530Ssam else 1585170530Ssam ipi_cpu(hostcpu, ipinum); 1586243882Sglebius} 1587170530Ssam 1588243882Sglebiusbool 1589170530Ssamvlapic_enabled(struct vlapic *vlapic) 1590170530Ssam{ 1591170530Ssam struct LAPIC *lapic = vlapic->apic_page; 1592170530Ssam 1593170530Ssam if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 && 1594170530Ssam (lapic->svr & APIC_SVR_ENABLE) != 0) 1595170530Ssam return (true); 1596170530Ssam else 1597170530Ssam return (false); 1598170530Ssam} 1599232480Sadrian 1600170530Ssamstatic void 1601170530Ssamvlapic_set_tmr(struct vlapic *vlapic, int vector, bool level) 1602170530Ssam{ 1603232480Sadrian struct LAPIC *lapic; 1604232480Sadrian uint32_t *tmrptr, mask; 1605232480Sadrian int idx; 1606232480Sadrian 1607232480Sadrian lapic = vlapic->apic_page; 1608232480Sadrian tmrptr = &lapic->tmr0; 1609232480Sadrian idx = (vector / 32) * 4; 1610232480Sadrian mask = 1 << (vector % 32); 1611232480Sadrian if (level) 1612232480Sadrian tmrptr[idx] |= mask; 1613170530Ssam else 1614170530Ssam tmrptr[idx] &= ~mask; 1615170530Ssam 1616170530Ssam if (vlapic->ops.set_tmr != NULL) 1617170530Ssam (*vlapic->ops.set_tmr)(vlapic, vector, level); 1618170530Ssam} 1619170530Ssam 1620250974Sadrianvoid 1621250974Sadrianvlapic_reset_tmr(struct vlapic *vlapic) 1622250974Sadrian{ 1623250974Sadrian int vector; 1624170530Ssam 1625170530Ssam VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered"); 1626170530Ssam 1627170530Ssam for (vector = 0; vector <= 255; vector++) 1628170530Ssam vlapic_set_tmr(vlapic, vector, false); 1629170530Ssam} 1630170530Ssam 1631170530Ssamvoid 1632170530Ssamvlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys, 1633170530Ssam int delmode, int vector) 1634188380Sweongyo{ 1635188380Sweongyo cpuset_t dmask; 1636188380Sweongyo bool lowprio; 1637170530Ssam 1638170530Ssam KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector)); 1639170530Ssam 1640170530Ssam /* 1641170530Ssam * A level trigger is valid only for fixed and lowprio delivery modes. 1642170530Ssam */ 1643178354Ssam if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) { 1644178354Ssam VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for " 1645170530Ssam "delivery-mode %d", delmode); 1646170530Ssam return; 1647170530Ssam } 1648170530Ssam 1649170530Ssam lowprio = (delmode == APIC_DELMODE_LOWPRIO); 1650170530Ssam vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false); 1651170530Ssam 1652170530Ssam if (!CPU_ISSET(vlapic->vcpuid, &dmask)) 1653170530Ssam return; 1654170530Ssam 1655170530Ssam VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector); 1656170530Ssam vlapic_set_tmr(vlapic, vector, true); 1657170530Ssam} 1658170530Ssam