26c26
< * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 262350 2014-02-23 00:46:05Z jhb $
---
> * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 266339 2014-05-17 19:11:08Z jhb $
30c30
< __FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 262350 2014-02-23 00:46:05Z jhb $");
---
> __FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 266339 2014-05-17 19:11:08Z jhb $");
40d39
< #include <machine/clock.h>
43a43,45
> #include <machine/clock.h>
> #include <machine/smp.h>
>
46c48
< #include "vmm_stat.h"
---
> #include "vmm_ipi.h"
48a51,52
> #include "vmm_stat.h"
>
49a54
> #include "vlapic_priv.h"
52,90d56
< #define VLAPIC_CTR0(vlapic, format) \
< VCPU_CTR0((vlapic)->vm, (vlapic)->vcpuid, format)
<
< #define VLAPIC_CTR1(vlapic, format, p1) \
< VCPU_CTR1((vlapic)->vm, (vlapic)->vcpuid, format, p1)
<
< #define VLAPIC_CTR2(vlapic, format, p1, p2) \
< VCPU_CTR2((vlapic)->vm, (vlapic)->vcpuid, format, p1, p2)
<
< #define VLAPIC_CTR_IRR(vlapic, msg) \
< do { \
< uint32_t *irrptr = &(vlapic)->apic.irr0; \
< irrptr[0] = irrptr[0]; /* silence compiler */ \
< VLAPIC_CTR1((vlapic), msg " irr0 0x%08x", irrptr[0 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr1 0x%08x", irrptr[1 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr2 0x%08x", irrptr[2 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr3 0x%08x", irrptr[3 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr4 0x%08x", irrptr[4 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr5 0x%08x", irrptr[5 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr6 0x%08x", irrptr[6 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr7 0x%08x", irrptr[7 << 2]); \
< } while (0)
<
< #define VLAPIC_CTR_ISR(vlapic, msg) \
< do { \
< uint32_t *isrptr = &(vlapic)->apic.isr0; \
< isrptr[0] = isrptr[0]; /* silence compiler */ \
< VLAPIC_CTR1((vlapic), msg " isr0 0x%08x", isrptr[0 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr1 0x%08x", isrptr[1 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr2 0x%08x", isrptr[2 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr3 0x%08x", isrptr[3 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr4 0x%08x", isrptr[4 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr5 0x%08x", isrptr[5 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr6 0x%08x", isrptr[6 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr7 0x%08x", isrptr[7 << 2]); \
< } while (0)
<
< static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
<
94d59
< #define VLAPIC_MAXLVT_ENTRIES (APIC_LVT_CMCI)
98,131d62
< enum boot_state {
< BS_INIT,
< BS_SIPI,
< BS_RUNNING
< };
<
< struct vlapic {
< struct vm *vm;
< int vcpuid;
<
< struct LAPIC apic;
<
< uint32_t esr_pending;
< int esr_firing;
<
< struct callout callout; /* vlapic timer */
< struct bintime timer_fire_bt; /* callout expiry time */
< struct bintime timer_freq_bt; /* timer frequency */
< struct bintime timer_period_bt; /* timer period */
< struct mtx timer_mtx;
<
< /*
< * The 'isrvec_stk' is a stack of vectors injected by the local apic.
< * A vector is popped from the stack when the processor does an EOI.
< * The vector on the top of the stack is used to compute the
< * Processor Priority in conjunction with the TPR.
< */
< uint8_t isrvec_stk[ISRVEC_STK_SIZE];
< int isrvec_stk_top;
<
< uint64_t msr_apicbase;
< enum boot_state boot_state;
< };
<
134,137c65,66
< * vlapic_callout_handler() and vcpu accesses to the following registers:
< * - initial count register aka icr_timer
< * - current count register aka ccr_timer
< * - divide config register aka dcr_timer
---
> * vlapic_callout_handler() and vcpu accesses to:
> * - timer_freq_bt, timer_period_bt, timer_fire_bt
139,141d67
< *
< * Note that the vlapic_callout_handler() does not write to any of these
< * registers so they can be safely read from the vcpu context without locking.
159,160c85,86
< static __inline uint32_t
< vlapic_get_ldr(struct vlapic *vlapic)
---
> static uint32_t
> x2apic_ldr(struct vlapic *vlapic)
162d87
< struct LAPIC *lapic;
166,173c91,94
< lapic = &vlapic->apic;
< if (x2apic(vlapic)) {
< apicid = vlapic_get_id(vlapic);
< ldr = 1 << (apicid & 0xf);
< ldr |= (apicid & 0xffff0) << 12;
< return (ldr);
< } else
< return (lapic->ldr);
---
> apicid = vlapic_get_id(vlapic);
> ldr = 1 << (apicid & 0xf);
> ldr |= (apicid & 0xffff0) << 12;
> return (ldr);
176,177c97,98
< static __inline uint32_t
< vlapic_get_dfr(struct vlapic *vlapic)
---
> void
> vlapic_dfr_write_handler(struct vlapic *vlapic)
181,193c102
< lapic = &vlapic->apic;
< if (x2apic(vlapic))
< return (0);
< else
< return (lapic->dfr);
< }
<
< static void
< vlapic_set_dfr(struct vlapic *vlapic, uint32_t data)
< {
< uint32_t dfr;
< struct LAPIC *lapic;
<
---
> lapic = vlapic->apic_page;
195c104,106
< VM_CTR1(vlapic->vm, "write to DFR in x2apic mode: %#x", data);
---
> VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
> lapic->dfr);
> lapic->dfr = 0;
199,201c110,113
< lapic = &vlapic->apic;
< dfr = (lapic->dfr & APIC_DFR_RESERVED) | (data & APIC_DFR_MODEL_MASK);
< if ((dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
---
> lapic->dfr &= APIC_DFR_MODEL_MASK;
> lapic->dfr |= APIC_DFR_RESERVED;
>
> if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
203c115
< else if ((dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
---
> else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
206,208c118
< VLAPIC_CTR1(vlapic, "vlapic DFR in Unknown Model %#x", dfr);
<
< lapic->dfr = dfr;
---
> VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
211,212c121,122
< static void
< vlapic_set_ldr(struct vlapic *vlapic, uint32_t data)
---
> void
> vlapic_ldr_write_handler(struct vlapic *vlapic)
215a126,127
> lapic = vlapic->apic_page;
>
218,219c130,135
< VLAPIC_CTR1(vlapic, "write to LDR in x2apic mode: %#x", data);
< return;
---
> VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
> lapic->ldr);
> lapic->ldr = x2apic_ldr(vlapic);
> } else {
> lapic->ldr &= ~APIC_LDR_RESERVED;
> VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
220a137
> }
222,224c139,149
< lapic = &vlapic->apic;
< lapic->ldr = data & ~APIC_LDR_RESERVED;
< VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
---
> void
> vlapic_id_write_handler(struct vlapic *vlapic)
> {
> struct LAPIC *lapic;
>
> /*
> * We don't allow the ID register to be modified so reset it back to
> * its default value.
> */
> lapic = vlapic->apic_page;
> lapic->id = vlapic_get_id(vlapic);
252,261d176
< static void
< vlapic_mask_lvts(uint32_t *lvts, int num_lvt)
< {
< int i;
< for (i = 0; i < num_lvt; i++) {
< *lvts |= APIC_LVT_M;
< lvts += 4;
< }
< }
<
280c195
< lapic = &vlapic->apic;
---
> lapic = vlapic->apic_page;
304,305c219,220
< static void
< vlapic_set_dcr(struct vlapic *vlapic, uint32_t dcr)
---
> void
> vlapic_dcr_write_handler(struct vlapic *vlapic)
310c225
< lapic = &vlapic->apic;
---
> lapic = vlapic->apic_page;
313,315c228,230
< lapic->dcr_timer = dcr;
< divisor = vlapic_timer_divisor(dcr);
< VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d", dcr, divisor);
---
> divisor = vlapic_timer_divisor(lapic->dcr_timer);
> VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
> lapic->dcr_timer, divisor);
330,331c245,246
< static void
< vlapic_update_errors(struct vlapic *vlapic)
---
> void
> vlapic_esr_write_handler(struct vlapic *vlapic)
333c248,250
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic;
>
> lapic = vlapic->apic_page;
338,339c255,256
< static void
< vlapic_reset(struct vlapic *vlapic)
---
> int
> vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
342,344c259,260
<
< lapic = &vlapic->apic;
< bzero(lapic, sizeof(struct LAPIC));
---
> uint32_t *irrptr, *tmrptr, mask;
> int idx;
346,352c262
< lapic->version = VLAPIC_VERSION;
< lapic->version |= (VLAPIC_MAXLVT_ENTRIES << MAXLVTSHIFT);
< lapic->dfr = 0xffffffff;
< lapic->svr = APIC_SVR_VECTOR;
< vlapic_mask_lvts(&lapic->lvt_timer, 6);
< vlapic_mask_lvts(&lapic->lvt_cmci, 1);
< vlapic_set_dcr(vlapic, 0);
---
> KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
354,369c264
< if (vlapic->vcpuid == 0)
< vlapic->boot_state = BS_RUNNING; /* BSP */
< else
< vlapic->boot_state = BS_INIT; /* AP */
< }
<
< void
< vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
< {
< struct LAPIC *lapic = &vlapic->apic;
< uint32_t *irrptr, *tmrptr, mask;
< int idx;
<
< if (vector < 0 || vector >= 256)
< panic("vlapic_set_intr_ready: invalid vector %d\n", vector);
<
---
> lapic = vlapic->apic_page;
373c268
< return;
---
> return (0);
378c273,275
< return;
---
> VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
> vector);
> return (1);
380c277,280
<
---
>
> if (vlapic->ops.set_intr_ready)
> return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
>
388,390c288,289
< * Upon acceptance of an interrupt into the IRR the corresponding
< * TMR bit is cleared for edge-triggered interrupts and set for
< * level-triggered interrupts.
---
> * Verify that the trigger-mode of the interrupt matches with
> * the vlapic TMR registers.
393,396c292,294
< if (level)
< atomic_set_int(&tmrptr[idx], mask);
< else
< atomic_clear_int(&tmrptr[idx], mask);
---
> KASSERT((tmrptr[idx] & mask) == (level ? mask : 0),
> ("vlapic TMR[%d] is 0x%08x but interrupt is %s-triggered",
> idx / 4, tmrptr[idx], level ? "level" : "edge"));
398a297
> return (1);
404c303
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
417a317,353
> static __inline int
> lvt_off_to_idx(uint32_t offset)
> {
> int index;
>
> switch (offset) {
> case APIC_OFFSET_CMCI_LVT:
> index = APIC_LVT_CMCI;
> break;
> case APIC_OFFSET_TIMER_LVT:
> index = APIC_LVT_TIMER;
> break;
> case APIC_OFFSET_THERM_LVT:
> index = APIC_LVT_THERMAL;
> break;
> case APIC_OFFSET_PERF_LVT:
> index = APIC_LVT_PMC;
> break;
> case APIC_OFFSET_LINT0_LVT:
> index = APIC_LVT_LINT0;
> break;
> case APIC_OFFSET_LINT1_LVT:
> index = APIC_LVT_LINT1;
> break;
> case APIC_OFFSET_ERROR_LVT:
> index = APIC_LVT_ERROR;
> break;
> default:
> index = -1;
> break;
> }
> KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
> "invalid lvt index %d for offset %#x", index, offset));
>
> return (index);
> }
>
420a357,358
> int idx;
> uint32_t val;
422c360,362
< return (*vlapic_get_lvtptr(vlapic, offset));
---
> idx = lvt_off_to_idx(offset);
> val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
> return (val);
425,426c365,366
< static void
< vlapic_set_lvt(struct vlapic *vlapic, uint32_t offset, uint32_t val)
---
> void
> vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
428c368
< uint32_t *lvtptr, mask;
---
> uint32_t *lvtptr, mask, val;
429a370
> int idx;
431c372
< lapic = &vlapic->apic;
---
> lapic = vlapic->apic_page;
432a374,375
> val = *lvtptr;
> idx = lvt_off_to_idx(offset);
434,436d376
< if (offset == APIC_OFFSET_TIMER_LVT)
< VLAPIC_TIMER_LOCK(vlapic);
<
454c394,397
< *lvtptr = val & mask;
---
> val &= mask;
> *lvtptr = val;
> atomic_store_rel_32(&vlapic->lvt_last[idx], val);
> }
456,457c399,423
< if (offset == APIC_OFFSET_TIMER_LVT)
< VLAPIC_TIMER_UNLOCK(vlapic);
---
> static void
> vlapic_mask_lvts(struct vlapic *vlapic)
> {
> struct LAPIC *lapic = vlapic->apic_page;
>
> lapic->lvt_cmci |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
>
> lapic->lvt_timer |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
>
> lapic->lvt_thermal |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
>
> lapic->lvt_pcint |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
>
> lapic->lvt_lint0 |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
>
> lapic->lvt_lint1 |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
>
> lapic->lvt_error |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
477,478c443,444
< vlapic_set_intr_ready(vlapic, vec, false);
< vcpu_notify_event(vlapic->vm, vlapic->vcpuid);
---
> if (vlapic_set_intr_ready(vlapic, vec, false))
> vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
497c463
< isrptr = &vlapic->apic.isr0;
---
> isrptr = &vlapic->apic_page->isr0;
522c488
< tpr = vlapic->apic.tpr;
---
> tpr = vlapic->apic_page->tpr;
551c517
< isrptr = &vlapic->apic.isr0;
---
> isrptr = &vlapic->apic_page->isr0;
571c537
< vlapic->apic.ppr = ppr;
---
> vlapic->apic_page->ppr = ppr;
578c544
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
678c644
< static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_ENTRIES,
---
> static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
738,739d703
< KASSERT(vlapic->apic.icr_timer != 0, ("vlapic timer is disabled"));
<
784,785c748,749
< static void
< vlapic_set_icr_timer(struct vlapic *vlapic, uint32_t icr_timer)
---
> void
> vlapic_icrtmr_write_handler(struct vlapic *vlapic)
788a753
> uint32_t icr_timer;
792,793c757,758
< lapic = &vlapic->apic;
< lapic->icr_timer = icr_timer;
---
> lapic = vlapic->apic_page;
> icr_timer = lapic->icr_timer;
875,876c840,841
< dfr = vlapic_get_dfr(vlapic);
< ldr = vlapic_get_ldr(vlapic);
---
> dfr = vlapic->apic_page->dfr;
> ldr = vlapic->apic_page->ldr;
915,916c880,881
< static int
< lapic_process_icr(struct vlapic *vlapic, uint64_t icrval, bool *retu)
---
> int
> vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
920a886
> uint64_t icrval;
924c890,895
<
---
> struct LAPIC *lapic;
>
> lapic = vlapic->apic_page;
> lapic->icr_lo &= ~APIC_DELSTAT_PEND;
> icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
>
933a905
> VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
936c908,910
<
---
>
> VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
>
966c940,942
< } else
---
> VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
> "to vcpuid %d", vec, i);
> } else {
967a944,946
> VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
> "to vcpuid %d", i);
> }
1022c1001
< vlapic_pending_intr(struct vlapic *vlapic)
---
> vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
1024c1003
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
1027a1007,1009
> if (vlapic->ops.pending_intr)
> return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
>
1042c1024,1026
< return (vector);
---
> if (vecptr != NULL)
> *vecptr = vector;
> return (1);
1047c1031
< return (-1);
---
> return (0);
1053c1037
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
1056a1041,1043
> if (vlapic->ops.intr_accepted)
> return ((*vlapic->ops.intr_accepted)(vlapic, vector));
>
1084,1085c1071,1072
< static void
< lapic_set_svr(struct vlapic *vlapic, uint32_t new)
---
> void
> vlapic_svr_write_handler(struct vlapic *vlapic)
1088c1075
< uint32_t old, changed;
---
> uint32_t old, new, changed;
1090,1091c1077,1082
< lapic = &vlapic->apic;
< old = lapic->svr;
---
> lapic = vlapic->apic_page;
>
> new = lapic->svr;
> old = vlapic->svr_last;
> vlapic->svr_last = new;
>
1096c1087,1088
< * The apic is now disabled so stop the apic timer.
---
> * The apic is now disabled so stop the apic timer
> * and mask all the LVT entries.
1101a1094
> vlapic_mask_lvts(vlapic);
1109c1102
< vlapic_set_icr_timer(vlapic, lapic->icr_timer);
---
> vlapic_icrtmr_write_handler(vlapic);
1112d1104
< lapic->svr = new;
1118c1110
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
1131c1123
< *data = vlapic_get_id(vlapic);
---
> *data = lapic->id;
1149c1141
< *data = vlapic_get_ldr(vlapic);
---
> *data = lapic->ldr;
1152c1144
< *data = vlapic_get_dfr(vlapic);
---
> *data = lapic->dfr;
1176a1169,1170
> if (x2apic(vlapic))
> *data |= (uint64_t)lapic->icr_hi << 32;
1183a1178,1182
> #ifdef INVARIANTS
> reg = vlapic_get_lvtptr(vlapic, offset);
> KASSERT(*data == *reg, ("inconsistent lvt value at "
> "offset %#lx: %#lx/%#x", offset, *data, *reg));
> #endif
1185c1184
< case APIC_OFFSET_ICR:
---
> case APIC_OFFSET_TIMER_ICR:
1188c1187
< case APIC_OFFSET_CCR:
---
> case APIC_OFFSET_TIMER_CCR:
1191c1190
< case APIC_OFFSET_DCR:
---
> case APIC_OFFSET_TIMER_DCR:
1207c1206,1207
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
> uint32_t *regptr;
1209a1210,1212
> KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
> ("vlapic_write: invalid offset %#lx", offset));
>
1217d1219
< offset &= ~3;
1220a1223,1224
> lapic->id = data;
> vlapic_id_write_handler(vlapic);
1230c1234,1235
< vlapic_set_ldr(vlapic, data);
---
> lapic->ldr = data;
> vlapic_ldr_write_handler(vlapic);
1233c1238,1239
< vlapic_set_dfr(vlapic, data);
---
> lapic->dfr = data;
> vlapic_dfr_write_handler(vlapic);
1236c1242,1243
< lapic_set_svr(vlapic, data);
---
> lapic->svr = data;
> vlapic_svr_write_handler(vlapic);
1239,1243c1246,1249
< if (!x2apic(vlapic)) {
< data &= 0xffffffff;
< data |= (uint64_t)lapic->icr_hi << 32;
< }
< retval = lapic_process_icr(vlapic, data, retu);
---
> lapic->icr_lo = data;
> if (x2apic(vlapic))
> lapic->icr_hi = data >> 32;
> retval = vlapic_icrlo_write_handler(vlapic, retu);
1246,1249c1252
< if (!x2apic(vlapic)) {
< retval = 0;
< lapic->icr_hi = data;
< }
---
> lapic->icr_hi = data;
1253c1256,1258
< vlapic_set_lvt(vlapic, offset, data);
---
> regptr = vlapic_get_lvtptr(vlapic, offset);
> *regptr = data;
> vlapic_lvt_write_handler(vlapic, offset);
1255,1256c1260,1262
< case APIC_OFFSET_ICR:
< vlapic_set_icr_timer(vlapic, data);
---
> case APIC_OFFSET_TIMER_ICR:
> lapic->icr_timer = data;
> vlapic_icrtmr_write_handler(vlapic);
1259,1260c1265,1267
< case APIC_OFFSET_DCR:
< vlapic_set_dcr(vlapic, data);
---
> case APIC_OFFSET_TIMER_DCR:
> lapic->dcr_timer = data;
> vlapic_dcr_write_handler(vlapic);
1264c1271
< vlapic_update_errors(vlapic);
---
> vlapic_esr_write_handler(vlapic);
1273c1280
< case APIC_OFFSET_CCR:
---
> case APIC_OFFSET_TIMER_CCR:
1282,1283c1289,1290
< struct vlapic *
< vlapic_init(struct vm *vm, int vcpuid)
---
> static void
> vlapic_reset(struct vlapic *vlapic)
1285c1292,1295
< struct vlapic *vlapic;
---
> struct LAPIC *lapic;
>
> lapic = vlapic->apic_page;
> bzero(lapic, sizeof(struct LAPIC));
1287,1289c1297,1303
< vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO);
< vlapic->vm = vm;
< vlapic->vcpuid = vcpuid;
---
> lapic->id = vlapic_get_id(vlapic);
> lapic->version = VLAPIC_VERSION;
> lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
> lapic->dfr = 0xffffffff;
> lapic->svr = APIC_SVR_VECTOR;
> vlapic_mask_lvts(vlapic);
> vlapic_reset_tmr(vlapic);
1290a1305,1324
> lapic->dcr_timer = 0;
> vlapic_dcr_write_handler(vlapic);
>
> if (vlapic->vcpuid == 0)
> vlapic->boot_state = BS_RUNNING; /* BSP */
> else
> vlapic->boot_state = BS_INIT; /* AP */
>
> vlapic->svr_last = lapic->svr;
> }
>
> void
> vlapic_init(struct vlapic *vlapic)
> {
> KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
> KASSERT(vlapic->vcpuid >= 0 && vlapic->vcpuid < VM_MAXCPU,
> ("vlapic_init: vcpuid is not initialized"));
> KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
> "initialized"));
>
1303c1337
< if (vcpuid == 0)
---
> if (vlapic->vcpuid == 0)
1307,1308d1340
<
< return (vlapic);
1316d1347
< free(vlapic, M_VLAPIC);
1327c1358
< vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val)
---
> vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
1329c1360
< int err;
---
> struct LAPIC *lapic;
1330a1362,1363
> uint64_t old;
> int err;
1337c1370
< val &= ~APICBASE_X2APIC;
---
> new &= ~APICBASE_X2APIC;
1339c1372,1389
< vlapic->msr_apicbase = val;
---
> old = vlapic->msr_apicbase;
> vlapic->msr_apicbase = new;
>
> /*
> * If the vlapic is switching between xAPIC and x2APIC modes then
> * reset the mode-dependent registers.
> */
> if ((old ^ new) & APICBASE_X2APIC) {
> lapic = vlapic->apic_page;
> lapic->id = vlapic_get_id(vlapic);
> if (x2apic(vlapic)) {
> lapic->ldr = x2apic_ldr(vlapic);
> lapic->dfr = 0;
> } else {
> lapic->ldr = 0;
> lapic->dfr = 0xffffffff;
> }
> }
1380a1431,1448
> void
> vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
> {
> /*
> * Post an interrupt to the vcpu currently running on 'hostcpu'.
> *
> * This is done by leveraging features like Posted Interrupts (Intel)
> * Doorbell MSR (AMD AVIC) that avoid a VM exit.
> *
> * If neither of these features are available then fallback to
> * sending an IPI to 'hostcpu'.
> */
> if (vlapic->ops.post_intr)
> (*vlapic->ops.post_intr)(vlapic, hostcpu);
> else
> ipi_cpu(hostcpu, ipinum);
> }
>
1384c1452
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
1391a1460,1518
>
> static void
> vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
> {
> struct LAPIC *lapic;
> uint32_t *tmrptr, mask;
> int idx;
>
> lapic = vlapic->apic_page;
> tmrptr = &lapic->tmr0;
> idx = (vector / 32) * 4;
> mask = 1 << (vector % 32);
> if (level)
> tmrptr[idx] |= mask;
> else
> tmrptr[idx] &= ~mask;
>
> if (vlapic->ops.set_tmr != NULL)
> (*vlapic->ops.set_tmr)(vlapic, vector, level);
> }
>
> void
> vlapic_reset_tmr(struct vlapic *vlapic)
> {
> int vector;
>
> VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
>
> for (vector = 0; vector <= 255; vector++)
> vlapic_set_tmr(vlapic, vector, false);
> }
>
> void
> vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
> int delmode, int vector)
> {
> cpuset_t dmask;
> bool lowprio;
>
> KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
>
> /*
> * A level trigger is valid only for fixed and lowprio delivery modes.
> */
> if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
> VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
> "delivery-mode %d", delmode);
> return;
> }
>
> lowprio = (delmode == APIC_DELMODE_LOWPRIO);
> vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
>
> if (!CPU_ISSET(vlapic->vcpuid, &dmask))
> return;
>
> VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
> vlapic_set_tmr(vlapic, vector, true);
> }
< * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 262350 2014-02-23 00:46:05Z jhb $
---
> * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 266339 2014-05-17 19:11:08Z jhb $
30c30
< __FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 262350 2014-02-23 00:46:05Z jhb $");
---
> __FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 266339 2014-05-17 19:11:08Z jhb $");
40d39
< #include <machine/clock.h>
43a43,45
> #include <machine/clock.h>
> #include <machine/smp.h>
>
46c48
< #include "vmm_stat.h"
---
> #include "vmm_ipi.h"
48a51,52
> #include "vmm_stat.h"
>
49a54
> #include "vlapic_priv.h"
52,90d56
< #define VLAPIC_CTR0(vlapic, format) \
< VCPU_CTR0((vlapic)->vm, (vlapic)->vcpuid, format)
<
< #define VLAPIC_CTR1(vlapic, format, p1) \
< VCPU_CTR1((vlapic)->vm, (vlapic)->vcpuid, format, p1)
<
< #define VLAPIC_CTR2(vlapic, format, p1, p2) \
< VCPU_CTR2((vlapic)->vm, (vlapic)->vcpuid, format, p1, p2)
<
< #define VLAPIC_CTR_IRR(vlapic, msg) \
< do { \
< uint32_t *irrptr = &(vlapic)->apic.irr0; \
< irrptr[0] = irrptr[0]; /* silence compiler */ \
< VLAPIC_CTR1((vlapic), msg " irr0 0x%08x", irrptr[0 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr1 0x%08x", irrptr[1 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr2 0x%08x", irrptr[2 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr3 0x%08x", irrptr[3 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr4 0x%08x", irrptr[4 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr5 0x%08x", irrptr[5 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr6 0x%08x", irrptr[6 << 2]); \
< VLAPIC_CTR1((vlapic), msg " irr7 0x%08x", irrptr[7 << 2]); \
< } while (0)
<
< #define VLAPIC_CTR_ISR(vlapic, msg) \
< do { \
< uint32_t *isrptr = &(vlapic)->apic.isr0; \
< isrptr[0] = isrptr[0]; /* silence compiler */ \
< VLAPIC_CTR1((vlapic), msg " isr0 0x%08x", isrptr[0 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr1 0x%08x", isrptr[1 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr2 0x%08x", isrptr[2 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr3 0x%08x", isrptr[3 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr4 0x%08x", isrptr[4 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr5 0x%08x", isrptr[5 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr6 0x%08x", isrptr[6 << 2]); \
< VLAPIC_CTR1((vlapic), msg " isr7 0x%08x", isrptr[7 << 2]); \
< } while (0)
<
< static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
<
94d59
< #define VLAPIC_MAXLVT_ENTRIES (APIC_LVT_CMCI)
98,131d62
< enum boot_state {
< BS_INIT,
< BS_SIPI,
< BS_RUNNING
< };
<
< struct vlapic {
< struct vm *vm;
< int vcpuid;
<
< struct LAPIC apic;
<
< uint32_t esr_pending;
< int esr_firing;
<
< struct callout callout; /* vlapic timer */
< struct bintime timer_fire_bt; /* callout expiry time */
< struct bintime timer_freq_bt; /* timer frequency */
< struct bintime timer_period_bt; /* timer period */
< struct mtx timer_mtx;
<
< /*
< * The 'isrvec_stk' is a stack of vectors injected by the local apic.
< * A vector is popped from the stack when the processor does an EOI.
< * The vector on the top of the stack is used to compute the
< * Processor Priority in conjunction with the TPR.
< */
< uint8_t isrvec_stk[ISRVEC_STK_SIZE];
< int isrvec_stk_top;
<
< uint64_t msr_apicbase;
< enum boot_state boot_state;
< };
<
134,137c65,66
< * vlapic_callout_handler() and vcpu accesses to the following registers:
< * - initial count register aka icr_timer
< * - current count register aka ccr_timer
< * - divide config register aka dcr_timer
---
> * vlapic_callout_handler() and vcpu accesses to:
> * - timer_freq_bt, timer_period_bt, timer_fire_bt
139,141d67
< *
< * Note that the vlapic_callout_handler() does not write to any of these
< * registers so they can be safely read from the vcpu context without locking.
159,160c85,86
< static __inline uint32_t
< vlapic_get_ldr(struct vlapic *vlapic)
---
> static uint32_t
> x2apic_ldr(struct vlapic *vlapic)
162d87
< struct LAPIC *lapic;
166,173c91,94
< lapic = &vlapic->apic;
< if (x2apic(vlapic)) {
< apicid = vlapic_get_id(vlapic);
< ldr = 1 << (apicid & 0xf);
< ldr |= (apicid & 0xffff0) << 12;
< return (ldr);
< } else
< return (lapic->ldr);
---
> apicid = vlapic_get_id(vlapic);
> ldr = 1 << (apicid & 0xf);
> ldr |= (apicid & 0xffff0) << 12;
> return (ldr);
176,177c97,98
< static __inline uint32_t
< vlapic_get_dfr(struct vlapic *vlapic)
---
> void
> vlapic_dfr_write_handler(struct vlapic *vlapic)
181,193c102
< lapic = &vlapic->apic;
< if (x2apic(vlapic))
< return (0);
< else
< return (lapic->dfr);
< }
<
< static void
< vlapic_set_dfr(struct vlapic *vlapic, uint32_t data)
< {
< uint32_t dfr;
< struct LAPIC *lapic;
<
---
> lapic = vlapic->apic_page;
195c104,106
< VM_CTR1(vlapic->vm, "write to DFR in x2apic mode: %#x", data);
---
> VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
> lapic->dfr);
> lapic->dfr = 0;
199,201c110,113
< lapic = &vlapic->apic;
< dfr = (lapic->dfr & APIC_DFR_RESERVED) | (data & APIC_DFR_MODEL_MASK);
< if ((dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
---
> lapic->dfr &= APIC_DFR_MODEL_MASK;
> lapic->dfr |= APIC_DFR_RESERVED;
>
> if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
203c115
< else if ((dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
---
> else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
206,208c118
< VLAPIC_CTR1(vlapic, "vlapic DFR in Unknown Model %#x", dfr);
<
< lapic->dfr = dfr;
---
> VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
211,212c121,122
< static void
< vlapic_set_ldr(struct vlapic *vlapic, uint32_t data)
---
> void
> vlapic_ldr_write_handler(struct vlapic *vlapic)
215a126,127
> lapic = vlapic->apic_page;
>
218,219c130,135
< VLAPIC_CTR1(vlapic, "write to LDR in x2apic mode: %#x", data);
< return;
---
> VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
> lapic->ldr);
> lapic->ldr = x2apic_ldr(vlapic);
> } else {
> lapic->ldr &= ~APIC_LDR_RESERVED;
> VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
220a137
> }
222,224c139,149
< lapic = &vlapic->apic;
< lapic->ldr = data & ~APIC_LDR_RESERVED;
< VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
---
> void
> vlapic_id_write_handler(struct vlapic *vlapic)
> {
> struct LAPIC *lapic;
>
> /*
> * We don't allow the ID register to be modified so reset it back to
> * its default value.
> */
> lapic = vlapic->apic_page;
> lapic->id = vlapic_get_id(vlapic);
252,261d176
< static void
< vlapic_mask_lvts(uint32_t *lvts, int num_lvt)
< {
< int i;
< for (i = 0; i < num_lvt; i++) {
< *lvts |= APIC_LVT_M;
< lvts += 4;
< }
< }
<
280c195
< lapic = &vlapic->apic;
---
> lapic = vlapic->apic_page;
304,305c219,220
< static void
< vlapic_set_dcr(struct vlapic *vlapic, uint32_t dcr)
---
> void
> vlapic_dcr_write_handler(struct vlapic *vlapic)
310c225
< lapic = &vlapic->apic;
---
> lapic = vlapic->apic_page;
313,315c228,230
< lapic->dcr_timer = dcr;
< divisor = vlapic_timer_divisor(dcr);
< VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d", dcr, divisor);
---
> divisor = vlapic_timer_divisor(lapic->dcr_timer);
> VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
> lapic->dcr_timer, divisor);
330,331c245,246
< static void
< vlapic_update_errors(struct vlapic *vlapic)
---
> void
> vlapic_esr_write_handler(struct vlapic *vlapic)
333c248,250
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic;
>
> lapic = vlapic->apic_page;
338,339c255,256
< static void
< vlapic_reset(struct vlapic *vlapic)
---
> int
> vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
342,344c259,260
<
< lapic = &vlapic->apic;
< bzero(lapic, sizeof(struct LAPIC));
---
> uint32_t *irrptr, *tmrptr, mask;
> int idx;
346,352c262
< lapic->version = VLAPIC_VERSION;
< lapic->version |= (VLAPIC_MAXLVT_ENTRIES << MAXLVTSHIFT);
< lapic->dfr = 0xffffffff;
< lapic->svr = APIC_SVR_VECTOR;
< vlapic_mask_lvts(&lapic->lvt_timer, 6);
< vlapic_mask_lvts(&lapic->lvt_cmci, 1);
< vlapic_set_dcr(vlapic, 0);
---
> KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
354,369c264
< if (vlapic->vcpuid == 0)
< vlapic->boot_state = BS_RUNNING; /* BSP */
< else
< vlapic->boot_state = BS_INIT; /* AP */
< }
<
< void
< vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
< {
< struct LAPIC *lapic = &vlapic->apic;
< uint32_t *irrptr, *tmrptr, mask;
< int idx;
<
< if (vector < 0 || vector >= 256)
< panic("vlapic_set_intr_ready: invalid vector %d\n", vector);
<
---
> lapic = vlapic->apic_page;
373c268
< return;
---
> return (0);
378c273,275
< return;
---
> VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
> vector);
> return (1);
380c277,280
<
---
>
> if (vlapic->ops.set_intr_ready)
> return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
>
388,390c288,289
< * Upon acceptance of an interrupt into the IRR the corresponding
< * TMR bit is cleared for edge-triggered interrupts and set for
< * level-triggered interrupts.
---
> * Verify that the trigger-mode of the interrupt matches with
> * the vlapic TMR registers.
393,396c292,294
< if (level)
< atomic_set_int(&tmrptr[idx], mask);
< else
< atomic_clear_int(&tmrptr[idx], mask);
---
> KASSERT((tmrptr[idx] & mask) == (level ? mask : 0),
> ("vlapic TMR[%d] is 0x%08x but interrupt is %s-triggered",
> idx / 4, tmrptr[idx], level ? "level" : "edge"));
398a297
> return (1);
404c303
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
417a317,353
> static __inline int
> lvt_off_to_idx(uint32_t offset)
> {
> int index;
>
> switch (offset) {
> case APIC_OFFSET_CMCI_LVT:
> index = APIC_LVT_CMCI;
> break;
> case APIC_OFFSET_TIMER_LVT:
> index = APIC_LVT_TIMER;
> break;
> case APIC_OFFSET_THERM_LVT:
> index = APIC_LVT_THERMAL;
> break;
> case APIC_OFFSET_PERF_LVT:
> index = APIC_LVT_PMC;
> break;
> case APIC_OFFSET_LINT0_LVT:
> index = APIC_LVT_LINT0;
> break;
> case APIC_OFFSET_LINT1_LVT:
> index = APIC_LVT_LINT1;
> break;
> case APIC_OFFSET_ERROR_LVT:
> index = APIC_LVT_ERROR;
> break;
> default:
> index = -1;
> break;
> }
> KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
> "invalid lvt index %d for offset %#x", index, offset));
>
> return (index);
> }
>
420a357,358
> int idx;
> uint32_t val;
422c360,362
< return (*vlapic_get_lvtptr(vlapic, offset));
---
> idx = lvt_off_to_idx(offset);
> val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
> return (val);
425,426c365,366
< static void
< vlapic_set_lvt(struct vlapic *vlapic, uint32_t offset, uint32_t val)
---
> void
> vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
428c368
< uint32_t *lvtptr, mask;
---
> uint32_t *lvtptr, mask, val;
429a370
> int idx;
431c372
< lapic = &vlapic->apic;
---
> lapic = vlapic->apic_page;
432a374,375
> val = *lvtptr;
> idx = lvt_off_to_idx(offset);
434,436d376
< if (offset == APIC_OFFSET_TIMER_LVT)
< VLAPIC_TIMER_LOCK(vlapic);
<
454c394,397
< *lvtptr = val & mask;
---
> val &= mask;
> *lvtptr = val;
> atomic_store_rel_32(&vlapic->lvt_last[idx], val);
> }
456,457c399,423
< if (offset == APIC_OFFSET_TIMER_LVT)
< VLAPIC_TIMER_UNLOCK(vlapic);
---
> static void
> vlapic_mask_lvts(struct vlapic *vlapic)
> {
> struct LAPIC *lapic = vlapic->apic_page;
>
> lapic->lvt_cmci |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
>
> lapic->lvt_timer |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
>
> lapic->lvt_thermal |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
>
> lapic->lvt_pcint |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
>
> lapic->lvt_lint0 |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
>
> lapic->lvt_lint1 |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
>
> lapic->lvt_error |= APIC_LVT_M;
> vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
477,478c443,444
< vlapic_set_intr_ready(vlapic, vec, false);
< vcpu_notify_event(vlapic->vm, vlapic->vcpuid);
---
> if (vlapic_set_intr_ready(vlapic, vec, false))
> vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
497c463
< isrptr = &vlapic->apic.isr0;
---
> isrptr = &vlapic->apic_page->isr0;
522c488
< tpr = vlapic->apic.tpr;
---
> tpr = vlapic->apic_page->tpr;
551c517
< isrptr = &vlapic->apic.isr0;
---
> isrptr = &vlapic->apic_page->isr0;
571c537
< vlapic->apic.ppr = ppr;
---
> vlapic->apic_page->ppr = ppr;
578c544
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
678c644
< static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_ENTRIES,
---
> static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
738,739d703
< KASSERT(vlapic->apic.icr_timer != 0, ("vlapic timer is disabled"));
<
784,785c748,749
< static void
< vlapic_set_icr_timer(struct vlapic *vlapic, uint32_t icr_timer)
---
> void
> vlapic_icrtmr_write_handler(struct vlapic *vlapic)
788a753
> uint32_t icr_timer;
792,793c757,758
< lapic = &vlapic->apic;
< lapic->icr_timer = icr_timer;
---
> lapic = vlapic->apic_page;
> icr_timer = lapic->icr_timer;
875,876c840,841
< dfr = vlapic_get_dfr(vlapic);
< ldr = vlapic_get_ldr(vlapic);
---
> dfr = vlapic->apic_page->dfr;
> ldr = vlapic->apic_page->ldr;
915,916c880,881
< static int
< lapic_process_icr(struct vlapic *vlapic, uint64_t icrval, bool *retu)
---
> int
> vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
920a886
> uint64_t icrval;
924c890,895
<
---
> struct LAPIC *lapic;
>
> lapic = vlapic->apic_page;
> lapic->icr_lo &= ~APIC_DELSTAT_PEND;
> icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
>
933a905
> VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
936c908,910
<
---
>
> VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
>
966c940,942
< } else
---
> VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
> "to vcpuid %d", vec, i);
> } else {
967a944,946
> VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
> "to vcpuid %d", i);
> }
1022c1001
< vlapic_pending_intr(struct vlapic *vlapic)
---
> vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
1024c1003
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
1027a1007,1009
> if (vlapic->ops.pending_intr)
> return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
>
1042c1024,1026
< return (vector);
---
> if (vecptr != NULL)
> *vecptr = vector;
> return (1);
1047c1031
< return (-1);
---
> return (0);
1053c1037
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
1056a1041,1043
> if (vlapic->ops.intr_accepted)
> return ((*vlapic->ops.intr_accepted)(vlapic, vector));
>
1084,1085c1071,1072
< static void
< lapic_set_svr(struct vlapic *vlapic, uint32_t new)
---
> void
> vlapic_svr_write_handler(struct vlapic *vlapic)
1088c1075
< uint32_t old, changed;
---
> uint32_t old, new, changed;
1090,1091c1077,1082
< lapic = &vlapic->apic;
< old = lapic->svr;
---
> lapic = vlapic->apic_page;
>
> new = lapic->svr;
> old = vlapic->svr_last;
> vlapic->svr_last = new;
>
1096c1087,1088
< * The apic is now disabled so stop the apic timer.
---
> * The apic is now disabled so stop the apic timer
> * and mask all the LVT entries.
1101a1094
> vlapic_mask_lvts(vlapic);
1109c1102
< vlapic_set_icr_timer(vlapic, lapic->icr_timer);
---
> vlapic_icrtmr_write_handler(vlapic);
1112d1104
< lapic->svr = new;
1118c1110
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
1131c1123
< *data = vlapic_get_id(vlapic);
---
> *data = lapic->id;
1149c1141
< *data = vlapic_get_ldr(vlapic);
---
> *data = lapic->ldr;
1152c1144
< *data = vlapic_get_dfr(vlapic);
---
> *data = lapic->dfr;
1176a1169,1170
> if (x2apic(vlapic))
> *data |= (uint64_t)lapic->icr_hi << 32;
1183a1178,1182
> #ifdef INVARIANTS
> reg = vlapic_get_lvtptr(vlapic, offset);
> KASSERT(*data == *reg, ("inconsistent lvt value at "
> "offset %#lx: %#lx/%#x", offset, *data, *reg));
> #endif
1185c1184
< case APIC_OFFSET_ICR:
---
> case APIC_OFFSET_TIMER_ICR:
1188c1187
< case APIC_OFFSET_CCR:
---
> case APIC_OFFSET_TIMER_CCR:
1191c1190
< case APIC_OFFSET_DCR:
---
> case APIC_OFFSET_TIMER_DCR:
1207c1206,1207
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
> uint32_t *regptr;
1209a1210,1212
> KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
> ("vlapic_write: invalid offset %#lx", offset));
>
1217d1219
< offset &= ~3;
1220a1223,1224
> lapic->id = data;
> vlapic_id_write_handler(vlapic);
1230c1234,1235
< vlapic_set_ldr(vlapic, data);
---
> lapic->ldr = data;
> vlapic_ldr_write_handler(vlapic);
1233c1238,1239
< vlapic_set_dfr(vlapic, data);
---
> lapic->dfr = data;
> vlapic_dfr_write_handler(vlapic);
1236c1242,1243
< lapic_set_svr(vlapic, data);
---
> lapic->svr = data;
> vlapic_svr_write_handler(vlapic);
1239,1243c1246,1249
< if (!x2apic(vlapic)) {
< data &= 0xffffffff;
< data |= (uint64_t)lapic->icr_hi << 32;
< }
< retval = lapic_process_icr(vlapic, data, retu);
---
> lapic->icr_lo = data;
> if (x2apic(vlapic))
> lapic->icr_hi = data >> 32;
> retval = vlapic_icrlo_write_handler(vlapic, retu);
1246,1249c1252
< if (!x2apic(vlapic)) {
< retval = 0;
< lapic->icr_hi = data;
< }
---
> lapic->icr_hi = data;
1253c1256,1258
< vlapic_set_lvt(vlapic, offset, data);
---
> regptr = vlapic_get_lvtptr(vlapic, offset);
> *regptr = data;
> vlapic_lvt_write_handler(vlapic, offset);
1255,1256c1260,1262
< case APIC_OFFSET_ICR:
< vlapic_set_icr_timer(vlapic, data);
---
> case APIC_OFFSET_TIMER_ICR:
> lapic->icr_timer = data;
> vlapic_icrtmr_write_handler(vlapic);
1259,1260c1265,1267
< case APIC_OFFSET_DCR:
< vlapic_set_dcr(vlapic, data);
---
> case APIC_OFFSET_TIMER_DCR:
> lapic->dcr_timer = data;
> vlapic_dcr_write_handler(vlapic);
1264c1271
< vlapic_update_errors(vlapic);
---
> vlapic_esr_write_handler(vlapic);
1273c1280
< case APIC_OFFSET_CCR:
---
> case APIC_OFFSET_TIMER_CCR:
1282,1283c1289,1290
< struct vlapic *
< vlapic_init(struct vm *vm, int vcpuid)
---
> static void
> vlapic_reset(struct vlapic *vlapic)
1285c1292,1295
< struct vlapic *vlapic;
---
> struct LAPIC *lapic;
>
> lapic = vlapic->apic_page;
> bzero(lapic, sizeof(struct LAPIC));
1287,1289c1297,1303
< vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO);
< vlapic->vm = vm;
< vlapic->vcpuid = vcpuid;
---
> lapic->id = vlapic_get_id(vlapic);
> lapic->version = VLAPIC_VERSION;
> lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
> lapic->dfr = 0xffffffff;
> lapic->svr = APIC_SVR_VECTOR;
> vlapic_mask_lvts(vlapic);
> vlapic_reset_tmr(vlapic);
1290a1305,1324
> lapic->dcr_timer = 0;
> vlapic_dcr_write_handler(vlapic);
>
> if (vlapic->vcpuid == 0)
> vlapic->boot_state = BS_RUNNING; /* BSP */
> else
> vlapic->boot_state = BS_INIT; /* AP */
>
> vlapic->svr_last = lapic->svr;
> }
>
> void
> vlapic_init(struct vlapic *vlapic)
> {
> KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
> KASSERT(vlapic->vcpuid >= 0 && vlapic->vcpuid < VM_MAXCPU,
> ("vlapic_init: vcpuid is not initialized"));
> KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
> "initialized"));
>
1303c1337
< if (vcpuid == 0)
---
> if (vlapic->vcpuid == 0)
1307,1308d1340
<
< return (vlapic);
1316d1347
< free(vlapic, M_VLAPIC);
1327c1358
< vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val)
---
> vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
1329c1360
< int err;
---
> struct LAPIC *lapic;
1330a1362,1363
> uint64_t old;
> int err;
1337c1370
< val &= ~APICBASE_X2APIC;
---
> new &= ~APICBASE_X2APIC;
1339c1372,1389
< vlapic->msr_apicbase = val;
---
> old = vlapic->msr_apicbase;
> vlapic->msr_apicbase = new;
>
> /*
> * If the vlapic is switching between xAPIC and x2APIC modes then
> * reset the mode-dependent registers.
> */
> if ((old ^ new) & APICBASE_X2APIC) {
> lapic = vlapic->apic_page;
> lapic->id = vlapic_get_id(vlapic);
> if (x2apic(vlapic)) {
> lapic->ldr = x2apic_ldr(vlapic);
> lapic->dfr = 0;
> } else {
> lapic->ldr = 0;
> lapic->dfr = 0xffffffff;
> }
> }
1380a1431,1448
> void
> vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
> {
> /*
> * Post an interrupt to the vcpu currently running on 'hostcpu'.
> *
> * This is done by leveraging features like Posted Interrupts (Intel)
> * Doorbell MSR (AMD AVIC) that avoid a VM exit.
> *
> * If neither of these features are available then fallback to
> * sending an IPI to 'hostcpu'.
> */
> if (vlapic->ops.post_intr)
> (*vlapic->ops.post_intr)(vlapic, hostcpu);
> else
> ipi_cpu(hostcpu, ipinum);
> }
>
1384c1452
< struct LAPIC *lapic = &vlapic->apic;
---
> struct LAPIC *lapic = vlapic->apic_page;
1391a1460,1518
>
> static void
> vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
> {
> struct LAPIC *lapic;
> uint32_t *tmrptr, mask;
> int idx;
>
> lapic = vlapic->apic_page;
> tmrptr = &lapic->tmr0;
> idx = (vector / 32) * 4;
> mask = 1 << (vector % 32);
> if (level)
> tmrptr[idx] |= mask;
> else
> tmrptr[idx] &= ~mask;
>
> if (vlapic->ops.set_tmr != NULL)
> (*vlapic->ops.set_tmr)(vlapic, vector, level);
> }
>
> void
> vlapic_reset_tmr(struct vlapic *vlapic)
> {
> int vector;
>
> VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
>
> for (vector = 0; vector <= 255; vector++)
> vlapic_set_tmr(vlapic, vector, false);
> }
>
> void
> vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
> int delmode, int vector)
> {
> cpuset_t dmask;
> bool lowprio;
>
> KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
>
> /*
> * A level trigger is valid only for fixed and lowprio delivery modes.
> */
> if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
> VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
> "delivery-mode %d", delmode);
> return;
> }
>
> lowprio = (delmode == APIC_DELMODE_LOWPRIO);
> vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
>
> if (!CPU_ISSET(vlapic->vcpuid, &dmask))
> return;
>
> VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
> vlapic_set_tmr(vlapic, vector, true);
> }