Deleted Added
sdiff udiff text old ( 262350 ) new ( 266339 )
full compact
vlapic.c (262350) vlapic.c (266339)
1/*-
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 9 unchanged lines hidden (view full) ---

18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
1/*-
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 9 unchanged lines hidden (view full) ---

18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 262350 2014-02-23 00:46:05Z jhb $
26 * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 266339 2014-05-17 19:11:08Z jhb $
27 */
28
29#include <sys/cdefs.h>
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 262350 2014-02-23 00:46:05Z jhb $");
30__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 266339 2014-05-17 19:11:08Z jhb $");
31
32#include <sys/param.h>
33#include <sys/lock.h>
34#include <sys/kernel.h>
35#include <sys/malloc.h>
36#include <sys/mutex.h>
37#include <sys/systm.h>
38#include <sys/smp.h>
39
31
32#include <sys/param.h>
33#include <sys/lock.h>
34#include <sys/kernel.h>
35#include <sys/malloc.h>
36#include <sys/mutex.h>
37#include <sys/systm.h>
38#include <sys/smp.h>
39
40#include <machine/clock.h>
41#include <x86/specialreg.h>
42#include <x86/apicreg.h>
43
40#include <x86/specialreg.h>
41#include <x86/apicreg.h>
42
43#include <machine/clock.h>
44#include <machine/smp.h>
45
44#include <machine/vmm.h>
45
46#include <machine/vmm.h>
47
46#include "vmm_stat.h"
48#include "vmm_ipi.h"
47#include "vmm_lapic.h"
48#include "vmm_ktr.h"
49#include "vmm_lapic.h"
50#include "vmm_ktr.h"
51#include "vmm_stat.h"
52
49#include "vlapic.h"
53#include "vlapic.h"
54#include "vlapic_priv.h"
50#include "vioapic.h"
51
55#include "vioapic.h"
56
52#define VLAPIC_CTR0(vlapic, format) \
53 VCPU_CTR0((vlapic)->vm, (vlapic)->vcpuid, format)
54
55#define VLAPIC_CTR1(vlapic, format, p1) \
56 VCPU_CTR1((vlapic)->vm, (vlapic)->vcpuid, format, p1)
57
58#define VLAPIC_CTR2(vlapic, format, p1, p2) \
59 VCPU_CTR2((vlapic)->vm, (vlapic)->vcpuid, format, p1, p2)
60
61#define VLAPIC_CTR_IRR(vlapic, msg) \
62do { \
63 uint32_t *irrptr = &(vlapic)->apic.irr0; \
64 irrptr[0] = irrptr[0]; /* silence compiler */ \
65 VLAPIC_CTR1((vlapic), msg " irr0 0x%08x", irrptr[0 << 2]); \
66 VLAPIC_CTR1((vlapic), msg " irr1 0x%08x", irrptr[1 << 2]); \
67 VLAPIC_CTR1((vlapic), msg " irr2 0x%08x", irrptr[2 << 2]); \
68 VLAPIC_CTR1((vlapic), msg " irr3 0x%08x", irrptr[3 << 2]); \
69 VLAPIC_CTR1((vlapic), msg " irr4 0x%08x", irrptr[4 << 2]); \
70 VLAPIC_CTR1((vlapic), msg " irr5 0x%08x", irrptr[5 << 2]); \
71 VLAPIC_CTR1((vlapic), msg " irr6 0x%08x", irrptr[6 << 2]); \
72 VLAPIC_CTR1((vlapic), msg " irr7 0x%08x", irrptr[7 << 2]); \
73} while (0)
74
75#define VLAPIC_CTR_ISR(vlapic, msg) \
76do { \
77 uint32_t *isrptr = &(vlapic)->apic.isr0; \
78 isrptr[0] = isrptr[0]; /* silence compiler */ \
79 VLAPIC_CTR1((vlapic), msg " isr0 0x%08x", isrptr[0 << 2]); \
80 VLAPIC_CTR1((vlapic), msg " isr1 0x%08x", isrptr[1 << 2]); \
81 VLAPIC_CTR1((vlapic), msg " isr2 0x%08x", isrptr[2 << 2]); \
82 VLAPIC_CTR1((vlapic), msg " isr3 0x%08x", isrptr[3 << 2]); \
83 VLAPIC_CTR1((vlapic), msg " isr4 0x%08x", isrptr[4 << 2]); \
84 VLAPIC_CTR1((vlapic), msg " isr5 0x%08x", isrptr[5 << 2]); \
85 VLAPIC_CTR1((vlapic), msg " isr6 0x%08x", isrptr[6 << 2]); \
86 VLAPIC_CTR1((vlapic), msg " isr7 0x%08x", isrptr[7 << 2]); \
87} while (0)
88
89static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
90
91#define PRIO(x) ((x) >> 4)
92
93#define VLAPIC_VERSION (16)
57#define PRIO(x) ((x) >> 4)
58
59#define VLAPIC_VERSION (16)
94#define VLAPIC_MAXLVT_ENTRIES (APIC_LVT_CMCI)
95
96#define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
97
60
61#define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
62
98enum boot_state {
99 BS_INIT,
100 BS_SIPI,
101 BS_RUNNING
102};
103
104struct vlapic {
105 struct vm *vm;
106 int vcpuid;
107
108 struct LAPIC apic;
109
110 uint32_t esr_pending;
111 int esr_firing;
112
113 struct callout callout; /* vlapic timer */
114 struct bintime timer_fire_bt; /* callout expiry time */
115 struct bintime timer_freq_bt; /* timer frequency */
116 struct bintime timer_period_bt; /* timer period */
117 struct mtx timer_mtx;
118
119 /*
120 * The 'isrvec_stk' is a stack of vectors injected by the local apic.
121 * A vector is popped from the stack when the processor does an EOI.
122 * The vector on the top of the stack is used to compute the
123 * Processor Priority in conjunction with the TPR.
124 */
125 uint8_t isrvec_stk[ISRVEC_STK_SIZE];
126 int isrvec_stk_top;
127
128 uint64_t msr_apicbase;
129 enum boot_state boot_state;
130};
131
132/*
133 * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
63/*
64 * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
134 * vlapic_callout_handler() and vcpu accesses to the following registers:
135 * - initial count register aka icr_timer
136 * - current count register aka ccr_timer
137 * - divide config register aka dcr_timer
65 * vlapic_callout_handler() and vcpu accesses to:
66 * - timer_freq_bt, timer_period_bt, timer_fire_bt
138 * - timer LVT register
67 * - timer LVT register
139 *
140 * Note that the vlapic_callout_handler() does not write to any of these
141 * registers so they can be safely read from the vcpu context without locking.
142 */
143#define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx))
144#define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx))
145#define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx))
146
147#define VLAPIC_BUS_FREQ tsc_freq
148
149static __inline uint32_t
150vlapic_get_id(struct vlapic *vlapic)
151{
152
153 if (x2apic(vlapic))
154 return (vlapic->vcpuid);
155 else
156 return (vlapic->vcpuid << 24);
157}
158
68 */
69#define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx))
70#define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx))
71#define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx))
72
73#define VLAPIC_BUS_FREQ tsc_freq
74
75static __inline uint32_t
76vlapic_get_id(struct vlapic *vlapic)
77{
78
79 if (x2apic(vlapic))
80 return (vlapic->vcpuid);
81 else
82 return (vlapic->vcpuid << 24);
83}
84
159static __inline uint32_t
160vlapic_get_ldr(struct vlapic *vlapic)
85static uint32_t
86x2apic_ldr(struct vlapic *vlapic)
161{
87{
162 struct LAPIC *lapic;
163 int apicid;
164 uint32_t ldr;
165
88 int apicid;
89 uint32_t ldr;
90
166 lapic = &vlapic->apic;
167 if (x2apic(vlapic)) {
168 apicid = vlapic_get_id(vlapic);
169 ldr = 1 << (apicid & 0xf);
170 ldr |= (apicid & 0xffff0) << 12;
171 return (ldr);
172 } else
173 return (lapic->ldr);
91 apicid = vlapic_get_id(vlapic);
92 ldr = 1 << (apicid & 0xf);
93 ldr |= (apicid & 0xffff0) << 12;
94 return (ldr);
174}
175
95}
96
176static __inline uint32_t
177vlapic_get_dfr(struct vlapic *vlapic)
97void
98vlapic_dfr_write_handler(struct vlapic *vlapic)
178{
179 struct LAPIC *lapic;
180
99{
100 struct LAPIC *lapic;
101
181 lapic = &vlapic->apic;
182 if (x2apic(vlapic))
183 return (0);
184 else
185 return (lapic->dfr);
186}
187
188static void
189vlapic_set_dfr(struct vlapic *vlapic, uint32_t data)
190{
191 uint32_t dfr;
192 struct LAPIC *lapic;
193
102 lapic = vlapic->apic_page;
194 if (x2apic(vlapic)) {
103 if (x2apic(vlapic)) {
195 VM_CTR1(vlapic->vm, "write to DFR in x2apic mode: %#x", data);
104 VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
105 lapic->dfr);
106 lapic->dfr = 0;
196 return;
197 }
198
107 return;
108 }
109
199 lapic = &vlapic->apic;
200 dfr = (lapic->dfr & APIC_DFR_RESERVED) | (data & APIC_DFR_MODEL_MASK);
201 if ((dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
110 lapic->dfr &= APIC_DFR_MODEL_MASK;
111 lapic->dfr |= APIC_DFR_RESERVED;
112
113 if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
202 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
114 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
203 else if ((dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
115 else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
204 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
205 else
116 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
117 else
206 VLAPIC_CTR1(vlapic, "vlapic DFR in Unknown Model %#x", dfr);
207
208 lapic->dfr = dfr;
118 VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
209}
210
119}
120
211static void
212vlapic_set_ldr(struct vlapic *vlapic, uint32_t data)
121void
122vlapic_ldr_write_handler(struct vlapic *vlapic)
213{
214 struct LAPIC *lapic;
215
123{
124 struct LAPIC *lapic;
125
126 lapic = vlapic->apic_page;
127
216 /* LDR is read-only in x2apic mode */
217 if (x2apic(vlapic)) {
128 /* LDR is read-only in x2apic mode */
129 if (x2apic(vlapic)) {
218 VLAPIC_CTR1(vlapic, "write to LDR in x2apic mode: %#x", data);
219 return;
130 VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
131 lapic->ldr);
132 lapic->ldr = x2apic_ldr(vlapic);
133 } else {
134 lapic->ldr &= ~APIC_LDR_RESERVED;
135 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
220 }
136 }
137}
221
138
222 lapic = &vlapic->apic;
223 lapic->ldr = data & ~APIC_LDR_RESERVED;
224 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
139void
140vlapic_id_write_handler(struct vlapic *vlapic)
141{
142 struct LAPIC *lapic;
143
144 /*
145 * We don't allow the ID register to be modified so reset it back to
146 * its default value.
147 */
148 lapic = vlapic->apic_page;
149 lapic->id = vlapic_get_id(vlapic);
225}
226
227static int
228vlapic_timer_divisor(uint32_t dcr)
229{
230 switch (dcr & 0xB) {
231 case APIC_TDCR_1:
232 return (1);

--- 11 unchanged lines hidden (view full) ---

244 return (64);
245 case APIC_TDCR_128:
246 return (128);
247 default:
248 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
249 }
250}
251
150}
151
152static int
153vlapic_timer_divisor(uint32_t dcr)
154{
155 switch (dcr & 0xB) {
156 case APIC_TDCR_1:
157 return (1);

--- 11 unchanged lines hidden (view full) ---

169 return (64);
170 case APIC_TDCR_128:
171 return (128);
172 default:
173 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
174 }
175}
176
252static void
253vlapic_mask_lvts(uint32_t *lvts, int num_lvt)
254{
255 int i;
256 for (i = 0; i < num_lvt; i++) {
257 *lvts |= APIC_LVT_M;
258 lvts += 4;
259 }
260}
261
262#if 0
263static inline void
264vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
265{
266 printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
267 *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
268 *lvt & APIC_LVTT_M);
269}
270#endif
271
272static uint32_t
273vlapic_get_ccr(struct vlapic *vlapic)
274{
275 struct bintime bt_now, bt_rem;
276 struct LAPIC *lapic;
277 uint32_t ccr;
278
279 ccr = 0;
177#if 0
178static inline void
179vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
180{
181 printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
182 *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
183 *lvt & APIC_LVTT_M);
184}
185#endif
186
187static uint32_t
188vlapic_get_ccr(struct vlapic *vlapic)
189{
190 struct bintime bt_now, bt_rem;
191 struct LAPIC *lapic;
192 uint32_t ccr;
193
194 ccr = 0;
280 lapic = &vlapic->apic;
195 lapic = vlapic->apic_page;
281
282 VLAPIC_TIMER_LOCK(vlapic);
283 if (callout_active(&vlapic->callout)) {
284 /*
285 * If the timer is scheduled to expire in the future then
286 * compute the value of 'ccr' based on the remaining time.
287 */
288 binuptime(&bt_now);

--- 7 unchanged lines hidden (view full) ---

296 KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
297 "icr_timer is %#x", ccr, lapic->icr_timer));
298 VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
299 ccr, lapic->icr_timer);
300 VLAPIC_TIMER_UNLOCK(vlapic);
301 return (ccr);
302}
303
196
197 VLAPIC_TIMER_LOCK(vlapic);
198 if (callout_active(&vlapic->callout)) {
199 /*
200 * If the timer is scheduled to expire in the future then
201 * compute the value of 'ccr' based on the remaining time.
202 */
203 binuptime(&bt_now);

--- 7 unchanged lines hidden (view full) ---

211 KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
212 "icr_timer is %#x", ccr, lapic->icr_timer));
213 VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
214 ccr, lapic->icr_timer);
215 VLAPIC_TIMER_UNLOCK(vlapic);
216 return (ccr);
217}
218
304static void
305vlapic_set_dcr(struct vlapic *vlapic, uint32_t dcr)
219void
220vlapic_dcr_write_handler(struct vlapic *vlapic)
306{
307 struct LAPIC *lapic;
308 int divisor;
309
221{
222 struct LAPIC *lapic;
223 int divisor;
224
310 lapic = &vlapic->apic;
225 lapic = vlapic->apic_page;
311 VLAPIC_TIMER_LOCK(vlapic);
312
226 VLAPIC_TIMER_LOCK(vlapic);
227
313 lapic->dcr_timer = dcr;
314 divisor = vlapic_timer_divisor(dcr);
315 VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d", dcr, divisor);
228 divisor = vlapic_timer_divisor(lapic->dcr_timer);
229 VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
230 lapic->dcr_timer, divisor);
316
317 /*
318 * Update the timer frequency and the timer period.
319 *
320 * XXX changes to the frequency divider will not take effect until
321 * the timer is reloaded.
322 */
323 FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
324 vlapic->timer_period_bt = vlapic->timer_freq_bt;
325 bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
326
327 VLAPIC_TIMER_UNLOCK(vlapic);
328}
329
231
232 /*
233 * Update the timer frequency and the timer period.
234 *
235 * XXX changes to the frequency divider will not take effect until
236 * the timer is reloaded.
237 */
238 FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
239 vlapic->timer_period_bt = vlapic->timer_freq_bt;
240 bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
241
242 VLAPIC_TIMER_UNLOCK(vlapic);
243}
244
330static void
331vlapic_update_errors(struct vlapic *vlapic)
245void
246vlapic_esr_write_handler(struct vlapic *vlapic)
332{
247{
333 struct LAPIC *lapic = &vlapic->apic;
248 struct LAPIC *lapic;
249
250 lapic = vlapic->apic_page;
334 lapic->esr = vlapic->esr_pending;
335 vlapic->esr_pending = 0;
336}
337
251 lapic->esr = vlapic->esr_pending;
252 vlapic->esr_pending = 0;
253}
254
338static void
339vlapic_reset(struct vlapic *vlapic)
255int
256vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
340{
341 struct LAPIC *lapic;
257{
258 struct LAPIC *lapic;
342
343 lapic = &vlapic->apic;
344 bzero(lapic, sizeof(struct LAPIC));
259 uint32_t *irrptr, *tmrptr, mask;
260 int idx;
345
261
346 lapic->version = VLAPIC_VERSION;
347 lapic->version |= (VLAPIC_MAXLVT_ENTRIES << MAXLVTSHIFT);
348 lapic->dfr = 0xffffffff;
349 lapic->svr = APIC_SVR_VECTOR;
350 vlapic_mask_lvts(&lapic->lvt_timer, 6);
351 vlapic_mask_lvts(&lapic->lvt_cmci, 1);
352 vlapic_set_dcr(vlapic, 0);
262 KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
353
263
354 if (vlapic->vcpuid == 0)
355 vlapic->boot_state = BS_RUNNING; /* BSP */
356 else
357 vlapic->boot_state = BS_INIT; /* AP */
358}
359
360void
361vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
362{
363 struct LAPIC *lapic = &vlapic->apic;
364 uint32_t *irrptr, *tmrptr, mask;
365 int idx;
366
367 if (vector < 0 || vector >= 256)
368 panic("vlapic_set_intr_ready: invalid vector %d\n", vector);
369
264 lapic = vlapic->apic_page;
370 if (!(lapic->svr & APIC_SVR_ENABLE)) {
371 VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
372 "interrupt %d", vector);
265 if (!(lapic->svr & APIC_SVR_ENABLE)) {
266 VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
267 "interrupt %d", vector);
373 return;
268 return (0);
374 }
375
376 if (vector < 16) {
377 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
269 }
270
271 if (vector < 16) {
272 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
378 return;
273 VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
274 vector);
275 return (1);
379 }
276 }
380
277
278 if (vlapic->ops.set_intr_ready)
279 return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
280
381 idx = (vector / 32) * 4;
382 mask = 1 << (vector % 32);
383
384 irrptr = &lapic->irr0;
385 atomic_set_int(&irrptr[idx], mask);
386
387 /*
281 idx = (vector / 32) * 4;
282 mask = 1 << (vector % 32);
283
284 irrptr = &lapic->irr0;
285 atomic_set_int(&irrptr[idx], mask);
286
287 /*
388 * Upon acceptance of an interrupt into the IRR the corresponding
389 * TMR bit is cleared for edge-triggered interrupts and set for
390 * level-triggered interrupts.
288 * Verify that the trigger-mode of the interrupt matches with
289 * the vlapic TMR registers.
391 */
392 tmrptr = &lapic->tmr0;
290 */
291 tmrptr = &lapic->tmr0;
393 if (level)
394 atomic_set_int(&tmrptr[idx], mask);
395 else
396 atomic_clear_int(&tmrptr[idx], mask);
292 KASSERT((tmrptr[idx] & mask) == (level ? mask : 0),
293 ("vlapic TMR[%d] is 0x%08x but interrupt is %s-triggered",
294 idx / 4, tmrptr[idx], level ? "level" : "edge"));
397
398 VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
295
296 VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
297 return (1);
399}
400
401static __inline uint32_t *
402vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
403{
298}
299
300static __inline uint32_t *
301vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
302{
404 struct LAPIC *lapic = &vlapic->apic;
303 struct LAPIC *lapic = vlapic->apic_page;
405 int i;
406
407 switch (offset) {
408 case APIC_OFFSET_CMCI_LVT:
409 return (&lapic->lvt_cmci);
410 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
411 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
412 return ((&lapic->lvt_timer) + i);;
413 default:
414 panic("vlapic_get_lvt: invalid LVT\n");
415 }
416}
417
304 int i;
305
306 switch (offset) {
307 case APIC_OFFSET_CMCI_LVT:
308 return (&lapic->lvt_cmci);
309 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
310 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
311 return ((&lapic->lvt_timer) + i);;
312 default:
313 panic("vlapic_get_lvt: invalid LVT\n");
314 }
315}
316
317static __inline int
318lvt_off_to_idx(uint32_t offset)
319{
320 int index;
321
322 switch (offset) {
323 case APIC_OFFSET_CMCI_LVT:
324 index = APIC_LVT_CMCI;
325 break;
326 case APIC_OFFSET_TIMER_LVT:
327 index = APIC_LVT_TIMER;
328 break;
329 case APIC_OFFSET_THERM_LVT:
330 index = APIC_LVT_THERMAL;
331 break;
332 case APIC_OFFSET_PERF_LVT:
333 index = APIC_LVT_PMC;
334 break;
335 case APIC_OFFSET_LINT0_LVT:
336 index = APIC_LVT_LINT0;
337 break;
338 case APIC_OFFSET_LINT1_LVT:
339 index = APIC_LVT_LINT1;
340 break;
341 case APIC_OFFSET_ERROR_LVT:
342 index = APIC_LVT_ERROR;
343 break;
344 default:
345 index = -1;
346 break;
347 }
348 KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
349 "invalid lvt index %d for offset %#x", index, offset));
350
351 return (index);
352}
353
418static __inline uint32_t
419vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
420{
354static __inline uint32_t
355vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
356{
357 int idx;
358 uint32_t val;
421
359
422 return (*vlapic_get_lvtptr(vlapic, offset));
360 idx = lvt_off_to_idx(offset);
361 val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
362 return (val);
423}
424
363}
364
425static void
426vlapic_set_lvt(struct vlapic *vlapic, uint32_t offset, uint32_t val)
365void
366vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
427{
367{
428 uint32_t *lvtptr, mask;
368 uint32_t *lvtptr, mask, val;
429 struct LAPIC *lapic;
369 struct LAPIC *lapic;
370 int idx;
430
371
431 lapic = &vlapic->apic;
372 lapic = vlapic->apic_page;
432 lvtptr = vlapic_get_lvtptr(vlapic, offset);
373 lvtptr = vlapic_get_lvtptr(vlapic, offset);
374 val = *lvtptr;
375 idx = lvt_off_to_idx(offset);
433
376
434 if (offset == APIC_OFFSET_TIMER_LVT)
435 VLAPIC_TIMER_LOCK(vlapic);
436
437 if (!(lapic->svr & APIC_SVR_ENABLE))
438 val |= APIC_LVT_M;
439 mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
440 switch (offset) {
441 case APIC_OFFSET_TIMER_LVT:
442 mask |= APIC_LVTT_TM;
443 break;
444 case APIC_OFFSET_ERROR_LVT:
445 break;
446 case APIC_OFFSET_LINT0_LVT:
447 case APIC_OFFSET_LINT1_LVT:
448 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
449 /* FALLTHROUGH */
450 default:
451 mask |= APIC_LVT_DM;
452 break;
453 }
377 if (!(lapic->svr & APIC_SVR_ENABLE))
378 val |= APIC_LVT_M;
379 mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
380 switch (offset) {
381 case APIC_OFFSET_TIMER_LVT:
382 mask |= APIC_LVTT_TM;
383 break;
384 case APIC_OFFSET_ERROR_LVT:
385 break;
386 case APIC_OFFSET_LINT0_LVT:
387 case APIC_OFFSET_LINT1_LVT:
388 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
389 /* FALLTHROUGH */
390 default:
391 mask |= APIC_LVT_DM;
392 break;
393 }
454 *lvtptr = val & mask;
394 val &= mask;
395 *lvtptr = val;
396 atomic_store_rel_32(&vlapic->lvt_last[idx], val);
397}
455
398
456 if (offset == APIC_OFFSET_TIMER_LVT)
457 VLAPIC_TIMER_UNLOCK(vlapic);
399static void
400vlapic_mask_lvts(struct vlapic *vlapic)
401{
402 struct LAPIC *lapic = vlapic->apic_page;
403
404 lapic->lvt_cmci |= APIC_LVT_M;
405 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
406
407 lapic->lvt_timer |= APIC_LVT_M;
408 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
409
410 lapic->lvt_thermal |= APIC_LVT_M;
411 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
412
413 lapic->lvt_pcint |= APIC_LVT_M;
414 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
415
416 lapic->lvt_lint0 |= APIC_LVT_M;
417 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
418
419 lapic->lvt_lint1 |= APIC_LVT_M;
420 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
421
422 lapic->lvt_error |= APIC_LVT_M;
423 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
458}
459
460static int
461vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
462{
463 uint32_t vec, mode;
464
465 if (lvt & APIC_LVT_M)
466 return (0);
467
468 vec = lvt & APIC_LVT_VECTOR;
469 mode = lvt & APIC_LVT_DM;
470
471 switch (mode) {
472 case APIC_LVT_DM_FIXED:
473 if (vec < 16) {
474 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
475 return (0);
476 }
424}
425
426static int
427vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
428{
429 uint32_t vec, mode;
430
431 if (lvt & APIC_LVT_M)
432 return (0);
433
434 vec = lvt & APIC_LVT_VECTOR;
435 mode = lvt & APIC_LVT_DM;
436
437 switch (mode) {
438 case APIC_LVT_DM_FIXED:
439 if (vec < 16) {
440 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
441 return (0);
442 }
477 vlapic_set_intr_ready(vlapic, vec, false);
478 vcpu_notify_event(vlapic->vm, vlapic->vcpuid);
443 if (vlapic_set_intr_ready(vlapic, vec, false))
444 vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
479 break;
480 case APIC_LVT_DM_NMI:
481 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
482 break;
483 default:
484 // Other modes ignored
485 return (0);
486 }
487 return (1);
488}
489
490#if 1
491static void
492dump_isrvec_stk(struct vlapic *vlapic)
493{
494 int i;
495 uint32_t *isrptr;
496
445 break;
446 case APIC_LVT_DM_NMI:
447 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
448 break;
449 default:
450 // Other modes ignored
451 return (0);
452 }
453 return (1);
454}
455
456#if 1
457static void
458dump_isrvec_stk(struct vlapic *vlapic)
459{
460 int i;
461 uint32_t *isrptr;
462
497 isrptr = &vlapic->apic.isr0;
463 isrptr = &vlapic->apic_page->isr0;
498 for (i = 0; i < 8; i++)
499 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
500
501 for (i = 0; i <= vlapic->isrvec_stk_top; i++)
502 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
503}
504#endif
505

--- 8 unchanged lines hidden (view full) ---

514
515 /*
516 * Note that the value on the stack at index 0 is always 0.
517 *
518 * This is a placeholder for the value of ISRV when none of the
519 * bits is set in the ISRx registers.
520 */
521 isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
464 for (i = 0; i < 8; i++)
465 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
466
467 for (i = 0; i <= vlapic->isrvec_stk_top; i++)
468 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
469}
470#endif
471

--- 8 unchanged lines hidden (view full) ---

480
481 /*
482 * Note that the value on the stack at index 0 is always 0.
483 *
484 * This is a placeholder for the value of ISRV when none of the
485 * bits is set in the ISRx registers.
486 */
487 isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
522 tpr = vlapic->apic.tpr;
488 tpr = vlapic->apic_page->tpr;
523
524#if 1
525 {
526 int i, lastprio, curprio, vector, idx;
527 uint32_t *isrptr;
528
529 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
530 panic("isrvec_stk is corrupted: %d", isrvec);

--- 12 unchanged lines hidden (view full) ---

543 lastprio = curprio;
544 }
545
546 /*
547 * Make sure that each bit set in the ISRx registers has a
548 * corresponding entry on the isrvec stack.
549 */
550 i = 1;
489
490#if 1
491 {
492 int i, lastprio, curprio, vector, idx;
493 uint32_t *isrptr;
494
495 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
496 panic("isrvec_stk is corrupted: %d", isrvec);

--- 12 unchanged lines hidden (view full) ---

509 lastprio = curprio;
510 }
511
512 /*
513 * Make sure that each bit set in the ISRx registers has a
514 * corresponding entry on the isrvec stack.
515 */
516 i = 1;
551 isrptr = &vlapic->apic.isr0;
517 isrptr = &vlapic->apic_page->isr0;
552 for (vector = 0; vector < 256; vector++) {
553 idx = (vector / 32) * 4;
554 if (isrptr[idx] & (1 << (vector % 32))) {
555 if (i > vlapic->isrvec_stk_top ||
556 vlapic->isrvec_stk[i] != vector) {
557 dump_isrvec_stk(vlapic);
558 panic("ISR and isrvec_stk out of sync");
559 }
560 i++;
561 }
562 }
563 }
564#endif
565
566 if (PRIO(tpr) >= PRIO(isrvec))
567 ppr = tpr;
568 else
569 ppr = isrvec & 0xf0;
570
518 for (vector = 0; vector < 256; vector++) {
519 idx = (vector / 32) * 4;
520 if (isrptr[idx] & (1 << (vector % 32))) {
521 if (i > vlapic->isrvec_stk_top ||
522 vlapic->isrvec_stk[i] != vector) {
523 dump_isrvec_stk(vlapic);
524 panic("ISR and isrvec_stk out of sync");
525 }
526 i++;
527 }
528 }
529 }
530#endif
531
532 if (PRIO(tpr) >= PRIO(isrvec))
533 ppr = tpr;
534 else
535 ppr = isrvec & 0xf0;
536
571 vlapic->apic.ppr = ppr;
537 vlapic->apic_page->ppr = ppr;
572 VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
573}
574
575static void
576vlapic_process_eoi(struct vlapic *vlapic)
577{
538 VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
539}
540
541static void
542vlapic_process_eoi(struct vlapic *vlapic)
543{
578 struct LAPIC *lapic = &vlapic->apic;
544 struct LAPIC *lapic = vlapic->apic_page;
579 uint32_t *isrptr, *tmrptr;
580 int i, idx, bitpos, vector;
581
582 isrptr = &lapic->isr0;
583 tmrptr = &lapic->tmr0;
584
585 /*
586 * The x86 architecture reserves the the first 32 vectors for use

--- 83 unchanged lines hidden (view full) ---

670 uint32_t lvt;
671
672 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
673 if (vlapic_fire_lvt(vlapic, lvt)) {
674 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
675 }
676}
677
545 uint32_t *isrptr, *tmrptr;
546 int i, idx, bitpos, vector;
547
548 isrptr = &lapic->isr0;
549 tmrptr = &lapic->tmr0;
550
551 /*
552 * The x86 architecture reserves the the first 32 vectors for use

--- 83 unchanged lines hidden (view full) ---

636 uint32_t lvt;
637
638 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
639 if (vlapic_fire_lvt(vlapic, lvt)) {
640 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
641 }
642}
643
678static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_ENTRIES,
644static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
679 "lvts triggered");
680
681int
682vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
683{
684 uint32_t lvt;
685
686 switch (vector) {

--- 43 unchanged lines hidden (view full) ---

730 if (callout_pending(&vlapic->callout)) /* callout was reset */
731 goto done;
732
733 if (!callout_active(&vlapic->callout)) /* callout was stopped */
734 goto done;
735
736 callout_deactivate(&vlapic->callout);
737
645 "lvts triggered");
646
647int
648vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
649{
650 uint32_t lvt;
651
652 switch (vector) {

--- 43 unchanged lines hidden (view full) ---

696 if (callout_pending(&vlapic->callout)) /* callout was reset */
697 goto done;
698
699 if (!callout_active(&vlapic->callout)) /* callout was stopped */
700 goto done;
701
702 callout_deactivate(&vlapic->callout);
703
738 KASSERT(vlapic->apic.icr_timer != 0, ("vlapic timer is disabled"));
739
740 vlapic_fire_timer(vlapic);
741
742 if (vlapic_periodic_timer(vlapic)) {
743 binuptime(&btnow);
744 KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
745 ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
746 btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
747 vlapic->timer_fire_bt.frac));

--- 28 unchanged lines hidden (view full) ---

776 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
777 callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
778 vlapic_callout_handler, vlapic, 0);
779 }
780done:
781 VLAPIC_TIMER_UNLOCK(vlapic);
782}
783
704 vlapic_fire_timer(vlapic);
705
706 if (vlapic_periodic_timer(vlapic)) {
707 binuptime(&btnow);
708 KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
709 ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
710 btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
711 vlapic->timer_fire_bt.frac));

--- 28 unchanged lines hidden (view full) ---

740 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
741 callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
742 vlapic_callout_handler, vlapic, 0);
743 }
744done:
745 VLAPIC_TIMER_UNLOCK(vlapic);
746}
747
784static void
785vlapic_set_icr_timer(struct vlapic *vlapic, uint32_t icr_timer)
748void
749vlapic_icrtmr_write_handler(struct vlapic *vlapic)
786{
787 struct LAPIC *lapic;
788 sbintime_t sbt;
750{
751 struct LAPIC *lapic;
752 sbintime_t sbt;
753 uint32_t icr_timer;
789
790 VLAPIC_TIMER_LOCK(vlapic);
791
754
755 VLAPIC_TIMER_LOCK(vlapic);
756
792 lapic = &vlapic->apic;
793 lapic->icr_timer = icr_timer;
757 lapic = vlapic->apic_page;
758 icr_timer = lapic->icr_timer;
794
795 vlapic->timer_period_bt = vlapic->timer_freq_bt;
796 bintime_mul(&vlapic->timer_period_bt, icr_timer);
797
798 if (icr_timer != 0) {
799 binuptime(&vlapic->timer_fire_bt);
800 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
801

--- 65 unchanged lines hidden (view full) ---

867 */
868 CPU_ZERO(dmask);
869 amask = vm_active_cpus(vm);
870 while ((vcpuid = CPU_FFS(&amask)) != 0) {
871 vcpuid--;
872 CPU_CLR(vcpuid, &amask);
873
874 vlapic = vm_lapic(vm, vcpuid);
759
760 vlapic->timer_period_bt = vlapic->timer_freq_bt;
761 bintime_mul(&vlapic->timer_period_bt, icr_timer);
762
763 if (icr_timer != 0) {
764 binuptime(&vlapic->timer_fire_bt);
765 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
766

--- 65 unchanged lines hidden (view full) ---

832 */
833 CPU_ZERO(dmask);
834 amask = vm_active_cpus(vm);
835 while ((vcpuid = CPU_FFS(&amask)) != 0) {
836 vcpuid--;
837 CPU_CLR(vcpuid, &amask);
838
839 vlapic = vm_lapic(vm, vcpuid);
875 dfr = vlapic_get_dfr(vlapic);
876 ldr = vlapic_get_ldr(vlapic);
840 dfr = vlapic->apic_page->dfr;
841 ldr = vlapic->apic_page->ldr;
877
878 if ((dfr & APIC_DFR_MODEL_MASK) ==
879 APIC_DFR_MODEL_FLAT) {
880 ldest = ldr >> 24;
881 mda_ldest = mda_flat_ldest;
882 } else if ((dfr & APIC_DFR_MODEL_MASK) ==
883 APIC_DFR_MODEL_CLUSTER) {
884 if (x2apic(vlapic)) {

--- 22 unchanged lines hidden (view full) ---

907 break;
908 }
909 }
910 }
911}
912
913static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
914
842
843 if ((dfr & APIC_DFR_MODEL_MASK) ==
844 APIC_DFR_MODEL_FLAT) {
845 ldest = ldr >> 24;
846 mda_ldest = mda_flat_ldest;
847 } else if ((dfr & APIC_DFR_MODEL_MASK) ==
848 APIC_DFR_MODEL_CLUSTER) {
849 if (x2apic(vlapic)) {

--- 22 unchanged lines hidden (view full) ---

872 break;
873 }
874 }
875 }
876}
877
878static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
879
915static int
916lapic_process_icr(struct vlapic *vlapic, uint64_t icrval, bool *retu)
880int
881vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
917{
918 int i;
919 bool phys;
920 cpuset_t dmask;
882{
883 int i;
884 bool phys;
885 cpuset_t dmask;
886 uint64_t icrval;
921 uint32_t dest, vec, mode;
922 struct vlapic *vlapic2;
923 struct vm_exit *vmexit;
887 uint32_t dest, vec, mode;
888 struct vlapic *vlapic2;
889 struct vm_exit *vmexit;
924
890 struct LAPIC *lapic;
891
892 lapic = vlapic->apic_page;
893 lapic->icr_lo &= ~APIC_DELSTAT_PEND;
894 icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
895
925 if (x2apic(vlapic))
926 dest = icrval >> 32;
927 else
928 dest = icrval >> (32 + 24);
929 vec = icrval & APIC_VECTOR_MASK;
930 mode = icrval & APIC_DELMODE_MASK;
931
932 if (mode == APIC_DELMODE_FIXED && vec < 16) {
933 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
896 if (x2apic(vlapic))
897 dest = icrval >> 32;
898 else
899 dest = icrval >> (32 + 24);
900 vec = icrval & APIC_VECTOR_MASK;
901 mode = icrval & APIC_DELMODE_MASK;
902
903 if (mode == APIC_DELMODE_FIXED && vec < 16) {
904 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
905 VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
934 return (0);
935 }
906 return (0);
907 }
936
908
909 VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
910
937 if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
938 switch (icrval & APIC_DEST_MASK) {
939 case APIC_DEST_DESTFLD:
940 phys = ((icrval & APIC_DESTMODE_LOG) == 0);
941 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
942 x2apic(vlapic));
943 break;
944 case APIC_DEST_SELF:

--- 13 unchanged lines hidden (view full) ---

958
959 while ((i = CPU_FFS(&dmask)) != 0) {
960 i--;
961 CPU_CLR(i, &dmask);
962 if (mode == APIC_DELMODE_FIXED) {
963 lapic_intr_edge(vlapic->vm, i, vec);
964 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
965 IPIS_SENT, i, 1);
911 if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
912 switch (icrval & APIC_DEST_MASK) {
913 case APIC_DEST_DESTFLD:
914 phys = ((icrval & APIC_DESTMODE_LOG) == 0);
915 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
916 x2apic(vlapic));
917 break;
918 case APIC_DEST_SELF:

--- 13 unchanged lines hidden (view full) ---

932
933 while ((i = CPU_FFS(&dmask)) != 0) {
934 i--;
935 CPU_CLR(i, &dmask);
936 if (mode == APIC_DELMODE_FIXED) {
937 lapic_intr_edge(vlapic->vm, i, vec);
938 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
939 IPIS_SENT, i, 1);
966 } else
940 VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
941 "to vcpuid %d", vec, i);
942 } else {
967 vm_inject_nmi(vlapic->vm, i);
943 vm_inject_nmi(vlapic->vm, i);
944 VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
945 "to vcpuid %d", i);
946 }
968 }
969
970 return (0); /* handled completely in the kernel */
971 }
972
973 if (mode == APIC_DELMODE_INIT) {
974 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
975 return (0);

--- 38 unchanged lines hidden (view full) ---

1014
1015 /*
1016 * This will cause a return to userland.
1017 */
1018 return (1);
1019}
1020
1021int
947 }
948
949 return (0); /* handled completely in the kernel */
950 }
951
952 if (mode == APIC_DELMODE_INIT) {
953 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
954 return (0);

--- 38 unchanged lines hidden (view full) ---

993
994 /*
995 * This will cause a return to userland.
996 */
997 return (1);
998}
999
1000int
1022vlapic_pending_intr(struct vlapic *vlapic)
1001vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
1023{
1002{
1024 struct LAPIC *lapic = &vlapic->apic;
1003 struct LAPIC *lapic = vlapic->apic_page;
1025 int idx, i, bitpos, vector;
1026 uint32_t *irrptr, val;
1027
1004 int idx, i, bitpos, vector;
1005 uint32_t *irrptr, val;
1006
1007 if (vlapic->ops.pending_intr)
1008 return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
1009
1028 irrptr = &lapic->irr0;
1029
1030 /*
1031 * The x86 architecture reserves the the first 32 vectors for use
1032 * by the processor.
1033 */
1034 for (i = 7; i > 0; i--) {
1035 idx = i * 4;
1036 val = atomic_load_acq_int(&irrptr[idx]);
1037 bitpos = fls(val);
1038 if (bitpos != 0) {
1039 vector = i * 32 + (bitpos - 1);
1040 if (PRIO(vector) > PRIO(lapic->ppr)) {
1041 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
1010 irrptr = &lapic->irr0;
1011
1012 /*
1013 * The x86 architecture reserves the the first 32 vectors for use
1014 * by the processor.
1015 */
1016 for (i = 7; i > 0; i--) {
1017 idx = i * 4;
1018 val = atomic_load_acq_int(&irrptr[idx]);
1019 bitpos = fls(val);
1020 if (bitpos != 0) {
1021 vector = i * 32 + (bitpos - 1);
1022 if (PRIO(vector) > PRIO(lapic->ppr)) {
1023 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
1042 return (vector);
1024 if (vecptr != NULL)
1025 *vecptr = vector;
1026 return (1);
1043 } else
1044 break;
1045 }
1046 }
1027 } else
1028 break;
1029 }
1030 }
1047 return (-1);
1031 return (0);
1048}
1049
1050void
1051vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1052{
1032}
1033
1034void
1035vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1036{
1053 struct LAPIC *lapic = &vlapic->apic;
1037 struct LAPIC *lapic = vlapic->apic_page;
1054 uint32_t *irrptr, *isrptr;
1055 int idx, stk_top;
1056
1038 uint32_t *irrptr, *isrptr;
1039 int idx, stk_top;
1040
1041 if (vlapic->ops.intr_accepted)
1042 return ((*vlapic->ops.intr_accepted)(vlapic, vector));
1043
1057 /*
1058 * clear the ready bit for vector being accepted in irr
1059 * and set the vector as in service in isr.
1060 */
1061 idx = (vector / 32) * 4;
1062
1063 irrptr = &lapic->irr0;
1064 atomic_clear_int(&irrptr[idx], 1 << (vector % 32));

--- 11 unchanged lines hidden (view full) ---

1076 stk_top = vlapic->isrvec_stk_top;
1077 if (stk_top >= ISRVEC_STK_SIZE)
1078 panic("isrvec_stk_top overflow %d", stk_top);
1079
1080 vlapic->isrvec_stk[stk_top] = vector;
1081 vlapic_update_ppr(vlapic);
1082}
1083
1044 /*
1045 * clear the ready bit for vector being accepted in irr
1046 * and set the vector as in service in isr.
1047 */
1048 idx = (vector / 32) * 4;
1049
1050 irrptr = &lapic->irr0;
1051 atomic_clear_int(&irrptr[idx], 1 << (vector % 32));

--- 11 unchanged lines hidden (view full) ---

1063 stk_top = vlapic->isrvec_stk_top;
1064 if (stk_top >= ISRVEC_STK_SIZE)
1065 panic("isrvec_stk_top overflow %d", stk_top);
1066
1067 vlapic->isrvec_stk[stk_top] = vector;
1068 vlapic_update_ppr(vlapic);
1069}
1070
1084static void
1085lapic_set_svr(struct vlapic *vlapic, uint32_t new)
1071void
1072vlapic_svr_write_handler(struct vlapic *vlapic)
1086{
1087 struct LAPIC *lapic;
1073{
1074 struct LAPIC *lapic;
1088 uint32_t old, changed;
1075 uint32_t old, new, changed;
1089
1076
1090 lapic = &vlapic->apic;
1091 old = lapic->svr;
1077 lapic = vlapic->apic_page;
1078
1079 new = lapic->svr;
1080 old = vlapic->svr_last;
1081 vlapic->svr_last = new;
1082
1092 changed = old ^ new;
1093 if ((changed & APIC_SVR_ENABLE) != 0) {
1094 if ((new & APIC_SVR_ENABLE) == 0) {
1095 /*
1083 changed = old ^ new;
1084 if ((changed & APIC_SVR_ENABLE) != 0) {
1085 if ((new & APIC_SVR_ENABLE) == 0) {
1086 /*
1096 * The apic is now disabled so stop the apic timer.
1087 * The apic is now disabled so stop the apic timer
1088 * and mask all the LVT entries.
1097 */
1098 VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1099 VLAPIC_TIMER_LOCK(vlapic);
1100 callout_stop(&vlapic->callout);
1101 VLAPIC_TIMER_UNLOCK(vlapic);
1089 */
1090 VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1091 VLAPIC_TIMER_LOCK(vlapic);
1092 callout_stop(&vlapic->callout);
1093 VLAPIC_TIMER_UNLOCK(vlapic);
1094 vlapic_mask_lvts(vlapic);
1102 } else {
1103 /*
1104 * The apic is now enabled so restart the apic timer
1105 * if it is configured in periodic mode.
1106 */
1107 VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1108 if (vlapic_periodic_timer(vlapic))
1095 } else {
1096 /*
1097 * The apic is now enabled so restart the apic timer
1098 * if it is configured in periodic mode.
1099 */
1100 VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1101 if (vlapic_periodic_timer(vlapic))
1109 vlapic_set_icr_timer(vlapic, lapic->icr_timer);
1102 vlapic_icrtmr_write_handler(vlapic);
1110 }
1111 }
1103 }
1104 }
1112 lapic->svr = new;
1113}
1114
1115int
1116vlapic_read(struct vlapic *vlapic, uint64_t offset, uint64_t *data, bool *retu)
1117{
1105}
1106
1107int
1108vlapic_read(struct vlapic *vlapic, uint64_t offset, uint64_t *data, bool *retu)
1109{
1118 struct LAPIC *lapic = &vlapic->apic;
1110 struct LAPIC *lapic = vlapic->apic_page;
1119 uint32_t *reg;
1120 int i;
1121
1122 if (offset > sizeof(*lapic)) {
1123 *data = 0;
1124 goto done;
1125 }
1126
1127 offset &= ~3;
1128 switch(offset)
1129 {
1130 case APIC_OFFSET_ID:
1111 uint32_t *reg;
1112 int i;
1113
1114 if (offset > sizeof(*lapic)) {
1115 *data = 0;
1116 goto done;
1117 }
1118
1119 offset &= ~3;
1120 switch(offset)
1121 {
1122 case APIC_OFFSET_ID:
1131 *data = vlapic_get_id(vlapic);
1123 *data = lapic->id;
1132 break;
1133 case APIC_OFFSET_VER:
1134 *data = lapic->version;
1135 break;
1136 case APIC_OFFSET_TPR:
1137 *data = lapic->tpr;
1138 break;
1139 case APIC_OFFSET_APR:
1140 *data = lapic->apr;
1141 break;
1142 case APIC_OFFSET_PPR:
1143 *data = lapic->ppr;
1144 break;
1145 case APIC_OFFSET_EOI:
1146 *data = lapic->eoi;
1147 break;
1148 case APIC_OFFSET_LDR:
1124 break;
1125 case APIC_OFFSET_VER:
1126 *data = lapic->version;
1127 break;
1128 case APIC_OFFSET_TPR:
1129 *data = lapic->tpr;
1130 break;
1131 case APIC_OFFSET_APR:
1132 *data = lapic->apr;
1133 break;
1134 case APIC_OFFSET_PPR:
1135 *data = lapic->ppr;
1136 break;
1137 case APIC_OFFSET_EOI:
1138 *data = lapic->eoi;
1139 break;
1140 case APIC_OFFSET_LDR:
1149 *data = vlapic_get_ldr(vlapic);
1141 *data = lapic->ldr;
1150 break;
1151 case APIC_OFFSET_DFR:
1142 break;
1143 case APIC_OFFSET_DFR:
1152 *data = vlapic_get_dfr(vlapic);
1144 *data = lapic->dfr;
1153 break;
1154 case APIC_OFFSET_SVR:
1155 *data = lapic->svr;
1156 break;
1157 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1158 i = (offset - APIC_OFFSET_ISR0) >> 2;
1159 reg = &lapic->isr0;
1160 *data = *(reg + i);

--- 8 unchanged lines hidden (view full) ---

1169 reg = &lapic->irr0;
1170 *data = atomic_load_acq_int(reg + i);
1171 break;
1172 case APIC_OFFSET_ESR:
1173 *data = lapic->esr;
1174 break;
1175 case APIC_OFFSET_ICR_LOW:
1176 *data = lapic->icr_lo;
1145 break;
1146 case APIC_OFFSET_SVR:
1147 *data = lapic->svr;
1148 break;
1149 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1150 i = (offset - APIC_OFFSET_ISR0) >> 2;
1151 reg = &lapic->isr0;
1152 *data = *(reg + i);

--- 8 unchanged lines hidden (view full) ---

1161 reg = &lapic->irr0;
1162 *data = atomic_load_acq_int(reg + i);
1163 break;
1164 case APIC_OFFSET_ESR:
1165 *data = lapic->esr;
1166 break;
1167 case APIC_OFFSET_ICR_LOW:
1168 *data = lapic->icr_lo;
1169 if (x2apic(vlapic))
1170 *data |= (uint64_t)lapic->icr_hi << 32;
1177 break;
1178 case APIC_OFFSET_ICR_HI:
1179 *data = lapic->icr_hi;
1180 break;
1181 case APIC_OFFSET_CMCI_LVT:
1182 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1183 *data = vlapic_get_lvt(vlapic, offset);
1171 break;
1172 case APIC_OFFSET_ICR_HI:
1173 *data = lapic->icr_hi;
1174 break;
1175 case APIC_OFFSET_CMCI_LVT:
1176 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1177 *data = vlapic_get_lvt(vlapic, offset);
1178#ifdef INVARIANTS
1179 reg = vlapic_get_lvtptr(vlapic, offset);
1180 KASSERT(*data == *reg, ("inconsistent lvt value at "
1181 "offset %#lx: %#lx/%#x", offset, *data, *reg));
1182#endif
1184 break;
1183 break;
1185 case APIC_OFFSET_ICR:
1184 case APIC_OFFSET_TIMER_ICR:
1186 *data = lapic->icr_timer;
1187 break;
1185 *data = lapic->icr_timer;
1186 break;
1188 case APIC_OFFSET_CCR:
1187 case APIC_OFFSET_TIMER_CCR:
1189 *data = vlapic_get_ccr(vlapic);
1190 break;
1188 *data = vlapic_get_ccr(vlapic);
1189 break;
1191 case APIC_OFFSET_DCR:
1190 case APIC_OFFSET_TIMER_DCR:
1192 *data = lapic->dcr_timer;
1193 break;
1194 case APIC_OFFSET_RRR:
1195 default:
1196 *data = 0;
1197 break;
1198 }
1199done:
1200 VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1201 return 0;
1202}
1203
1204int
1205vlapic_write(struct vlapic *vlapic, uint64_t offset, uint64_t data, bool *retu)
1206{
1191 *data = lapic->dcr_timer;
1192 break;
1193 case APIC_OFFSET_RRR:
1194 default:
1195 *data = 0;
1196 break;
1197 }
1198done:
1199 VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1200 return 0;
1201}
1202
1203int
1204vlapic_write(struct vlapic *vlapic, uint64_t offset, uint64_t data, bool *retu)
1205{
1207 struct LAPIC *lapic = &vlapic->apic;
1206 struct LAPIC *lapic = vlapic->apic_page;
1207 uint32_t *regptr;
1208 int retval;
1209
1208 int retval;
1209
1210 KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
1211 ("vlapic_write: invalid offset %#lx", offset));
1212
1210 VLAPIC_CTR2(vlapic, "vlapic write offset %#x, data %#lx", offset, data);
1211
1212 if (offset > sizeof(*lapic)) {
1213 return 0;
1214 }
1215
1216 retval = 0;
1213 VLAPIC_CTR2(vlapic, "vlapic write offset %#x, data %#lx", offset, data);
1214
1215 if (offset > sizeof(*lapic)) {
1216 return 0;
1217 }
1218
1219 retval = 0;
1217 offset &= ~3;
1218 switch(offset)
1219 {
1220 case APIC_OFFSET_ID:
1220 switch(offset)
1221 {
1222 case APIC_OFFSET_ID:
1223 lapic->id = data;
1224 vlapic_id_write_handler(vlapic);
1221 break;
1222 case APIC_OFFSET_TPR:
1223 lapic->tpr = data & 0xff;
1224 vlapic_update_ppr(vlapic);
1225 break;
1226 case APIC_OFFSET_EOI:
1227 vlapic_process_eoi(vlapic);
1228 break;
1229 case APIC_OFFSET_LDR:
1225 break;
1226 case APIC_OFFSET_TPR:
1227 lapic->tpr = data & 0xff;
1228 vlapic_update_ppr(vlapic);
1229 break;
1230 case APIC_OFFSET_EOI:
1231 vlapic_process_eoi(vlapic);
1232 break;
1233 case APIC_OFFSET_LDR:
1230 vlapic_set_ldr(vlapic, data);
1234 lapic->ldr = data;
1235 vlapic_ldr_write_handler(vlapic);
1231 break;
1232 case APIC_OFFSET_DFR:
1236 break;
1237 case APIC_OFFSET_DFR:
1233 vlapic_set_dfr(vlapic, data);
1238 lapic->dfr = data;
1239 vlapic_dfr_write_handler(vlapic);
1234 break;
1235 case APIC_OFFSET_SVR:
1240 break;
1241 case APIC_OFFSET_SVR:
1236 lapic_set_svr(vlapic, data);
1242 lapic->svr = data;
1243 vlapic_svr_write_handler(vlapic);
1237 break;
1238 case APIC_OFFSET_ICR_LOW:
1244 break;
1245 case APIC_OFFSET_ICR_LOW:
1239 if (!x2apic(vlapic)) {
1240 data &= 0xffffffff;
1241 data |= (uint64_t)lapic->icr_hi << 32;
1242 }
1243 retval = lapic_process_icr(vlapic, data, retu);
1246 lapic->icr_lo = data;
1247 if (x2apic(vlapic))
1248 lapic->icr_hi = data >> 32;
1249 retval = vlapic_icrlo_write_handler(vlapic, retu);
1244 break;
1245 case APIC_OFFSET_ICR_HI:
1250 break;
1251 case APIC_OFFSET_ICR_HI:
1246 if (!x2apic(vlapic)) {
1247 retval = 0;
1248 lapic->icr_hi = data;
1249 }
1252 lapic->icr_hi = data;
1250 break;
1251 case APIC_OFFSET_CMCI_LVT:
1252 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1253 break;
1254 case APIC_OFFSET_CMCI_LVT:
1255 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1253 vlapic_set_lvt(vlapic, offset, data);
1256 regptr = vlapic_get_lvtptr(vlapic, offset);
1257 *regptr = data;
1258 vlapic_lvt_write_handler(vlapic, offset);
1254 break;
1259 break;
1255 case APIC_OFFSET_ICR:
1256 vlapic_set_icr_timer(vlapic, data);
1260 case APIC_OFFSET_TIMER_ICR:
1261 lapic->icr_timer = data;
1262 vlapic_icrtmr_write_handler(vlapic);
1257 break;
1258
1263 break;
1264
1259 case APIC_OFFSET_DCR:
1260 vlapic_set_dcr(vlapic, data);
1265 case APIC_OFFSET_TIMER_DCR:
1266 lapic->dcr_timer = data;
1267 vlapic_dcr_write_handler(vlapic);
1261 break;
1262
1263 case APIC_OFFSET_ESR:
1268 break;
1269
1270 case APIC_OFFSET_ESR:
1264 vlapic_update_errors(vlapic);
1271 vlapic_esr_write_handler(vlapic);
1265 break;
1266 case APIC_OFFSET_VER:
1267 case APIC_OFFSET_APR:
1268 case APIC_OFFSET_PPR:
1269 case APIC_OFFSET_RRR:
1270 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1271 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1272 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1272 break;
1273 case APIC_OFFSET_VER:
1274 case APIC_OFFSET_APR:
1275 case APIC_OFFSET_PPR:
1276 case APIC_OFFSET_RRR:
1277 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1278 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1279 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1273 case APIC_OFFSET_CCR:
1280 case APIC_OFFSET_TIMER_CCR:
1274 default:
1275 // Read only.
1276 break;
1277 }
1278
1279 return (retval);
1280}
1281
1281 default:
1282 // Read only.
1283 break;
1284 }
1285
1286 return (retval);
1287}
1288
1282struct vlapic *
1283vlapic_init(struct vm *vm, int vcpuid)
1289static void
1290vlapic_reset(struct vlapic *vlapic)
1284{
1291{
1285 struct vlapic *vlapic;
1292 struct LAPIC *lapic;
1293
1294 lapic = vlapic->apic_page;
1295 bzero(lapic, sizeof(struct LAPIC));
1286
1296
1287 vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO);
1288 vlapic->vm = vm;
1289 vlapic->vcpuid = vcpuid;
1297 lapic->id = vlapic_get_id(vlapic);
1298 lapic->version = VLAPIC_VERSION;
1299 lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
1300 lapic->dfr = 0xffffffff;
1301 lapic->svr = APIC_SVR_VECTOR;
1302 vlapic_mask_lvts(vlapic);
1303 vlapic_reset_tmr(vlapic);
1290
1304
1305 lapic->dcr_timer = 0;
1306 vlapic_dcr_write_handler(vlapic);
1307
1308 if (vlapic->vcpuid == 0)
1309 vlapic->boot_state = BS_RUNNING; /* BSP */
1310 else
1311 vlapic->boot_state = BS_INIT; /* AP */
1312
1313 vlapic->svr_last = lapic->svr;
1314}
1315
1316void
1317vlapic_init(struct vlapic *vlapic)
1318{
1319 KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
1320 KASSERT(vlapic->vcpuid >= 0 && vlapic->vcpuid < VM_MAXCPU,
1321 ("vlapic_init: vcpuid is not initialized"));
1322 KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
1323 "initialized"));
1324
1291 /*
1292 * If the vlapic is configured in x2apic mode then it will be
1293 * accessed in the critical section via the MSR emulation code.
1294 *
1295 * Therefore the timer mutex must be a spinlock because blockable
1296 * mutexes cannot be acquired in a critical section.
1297 */
1298 mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1299 callout_init(&vlapic->callout, 1);
1300
1301 vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1302
1325 /*
1326 * If the vlapic is configured in x2apic mode then it will be
1327 * accessed in the critical section via the MSR emulation code.
1328 *
1329 * Therefore the timer mutex must be a spinlock because blockable
1330 * mutexes cannot be acquired in a critical section.
1331 */
1332 mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1333 callout_init(&vlapic->callout, 1);
1334
1335 vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1336
1303 if (vcpuid == 0)
1337 if (vlapic->vcpuid == 0)
1304 vlapic->msr_apicbase |= APICBASE_BSP;
1305
1306 vlapic_reset(vlapic);
1338 vlapic->msr_apicbase |= APICBASE_BSP;
1339
1340 vlapic_reset(vlapic);
1307
1308 return (vlapic);
1309}
1310
1311void
1312vlapic_cleanup(struct vlapic *vlapic)
1313{
1314
1315 callout_drain(&vlapic->callout);
1341}
1342
1343void
1344vlapic_cleanup(struct vlapic *vlapic)
1345{
1346
1347 callout_drain(&vlapic->callout);
1316 free(vlapic, M_VLAPIC);
1317}
1318
1319uint64_t
1320vlapic_get_apicbase(struct vlapic *vlapic)
1321{
1322
1323 return (vlapic->msr_apicbase);
1324}
1325
1326void
1348}
1349
1350uint64_t
1351vlapic_get_apicbase(struct vlapic *vlapic)
1352{
1353
1354 return (vlapic->msr_apicbase);
1355}
1356
1357void
1327vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val)
1358vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
1328{
1359{
1329 int err;
1360 struct LAPIC *lapic;
1330 enum x2apic_state state;
1361 enum x2apic_state state;
1362 uint64_t old;
1363 int err;
1331
1332 err = vm_get_x2apic_state(vlapic->vm, vlapic->vcpuid, &state);
1333 if (err)
1334 panic("vlapic_set_apicbase: err %d fetching x2apic state", err);
1335
1336 if (state == X2APIC_DISABLED)
1364
1365 err = vm_get_x2apic_state(vlapic->vm, vlapic->vcpuid, &state);
1366 if (err)
1367 panic("vlapic_set_apicbase: err %d fetching x2apic state", err);
1368
1369 if (state == X2APIC_DISABLED)
1337 val &= ~APICBASE_X2APIC;
1370 new &= ~APICBASE_X2APIC;
1338
1371
1339 vlapic->msr_apicbase = val;
1372 old = vlapic->msr_apicbase;
1373 vlapic->msr_apicbase = new;
1374
1375 /*
1376 * If the vlapic is switching between xAPIC and x2APIC modes then
1377 * reset the mode-dependent registers.
1378 */
1379 if ((old ^ new) & APICBASE_X2APIC) {
1380 lapic = vlapic->apic_page;
1381 lapic->id = vlapic_get_id(vlapic);
1382 if (x2apic(vlapic)) {
1383 lapic->ldr = x2apic_ldr(vlapic);
1384 lapic->dfr = 0;
1385 } else {
1386 lapic->ldr = 0;
1387 lapic->dfr = 0xffffffff;
1388 }
1389 }
1340}
1341
1342void
1343vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1344{
1345 struct vlapic *vlapic;
1346
1347 vlapic = vm_lapic(vm, vcpuid);

--- 25 unchanged lines hidden (view full) ---

1373
1374 while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1375 vcpuid--;
1376 CPU_CLR(vcpuid, &dmask);
1377 lapic_set_intr(vm, vcpuid, vec, level);
1378 }
1379}
1380
1390}
1391
1392void
1393vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1394{
1395 struct vlapic *vlapic;
1396
1397 vlapic = vm_lapic(vm, vcpuid);

--- 25 unchanged lines hidden (view full) ---

1423
1424 while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1425 vcpuid--;
1426 CPU_CLR(vcpuid, &dmask);
1427 lapic_set_intr(vm, vcpuid, vec, level);
1428 }
1429}
1430
1431void
1432vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
1433{
1434 /*
1435 * Post an interrupt to the vcpu currently running on 'hostcpu'.
1436 *
1437 * This is done by leveraging features like Posted Interrupts (Intel)
1438 * Doorbell MSR (AMD AVIC) that avoid a VM exit.
1439 *
1440 * If neither of these features are available then fallback to
1441 * sending an IPI to 'hostcpu'.
1442 */
1443 if (vlapic->ops.post_intr)
1444 (*vlapic->ops.post_intr)(vlapic, hostcpu);
1445 else
1446 ipi_cpu(hostcpu, ipinum);
1447}
1448
1381bool
1382vlapic_enabled(struct vlapic *vlapic)
1383{
1449bool
1450vlapic_enabled(struct vlapic *vlapic)
1451{
1384 struct LAPIC *lapic = &vlapic->apic;
1452 struct LAPIC *lapic = vlapic->apic_page;
1385
1386 if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1387 (lapic->svr & APIC_SVR_ENABLE) != 0)
1388 return (true);
1389 else
1390 return (false);
1391}
1453
1454 if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1455 (lapic->svr & APIC_SVR_ENABLE) != 0)
1456 return (true);
1457 else
1458 return (false);
1459}
1460
1461static void
1462vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
1463{
1464 struct LAPIC *lapic;
1465 uint32_t *tmrptr, mask;
1466 int idx;
1467
1468 lapic = vlapic->apic_page;
1469 tmrptr = &lapic->tmr0;
1470 idx = (vector / 32) * 4;
1471 mask = 1 << (vector % 32);
1472 if (level)
1473 tmrptr[idx] |= mask;
1474 else
1475 tmrptr[idx] &= ~mask;
1476
1477 if (vlapic->ops.set_tmr != NULL)
1478 (*vlapic->ops.set_tmr)(vlapic, vector, level);
1479}
1480
1481void
1482vlapic_reset_tmr(struct vlapic *vlapic)
1483{
1484 int vector;
1485
1486 VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
1487
1488 for (vector = 0; vector <= 255; vector++)
1489 vlapic_set_tmr(vlapic, vector, false);
1490}
1491
1492void
1493vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
1494 int delmode, int vector)
1495{
1496 cpuset_t dmask;
1497 bool lowprio;
1498
1499 KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
1500
1501 /*
1502 * A level trigger is valid only for fixed and lowprio delivery modes.
1503 */
1504 if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
1505 VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
1506 "delivery-mode %d", delmode);
1507 return;
1508 }
1509
1510 lowprio = (delmode == APIC_DELMODE_LOWPRIO);
1511 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
1512
1513 if (!CPU_ISSET(vlapic->vcpuid, &dmask))
1514 return;
1515
1516 VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
1517 vlapic_set_tmr(vlapic, vector, true);
1518}