1/*-
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
| 1/*-
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
|
26 * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 262350 2014-02-23 00:46:05Z jhb $
| 26 * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 266339 2014-05-17 19:11:08Z jhb $
|
27 */
28
29#include <sys/cdefs.h>
| 27 */
28
29#include <sys/cdefs.h>
|
30__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 262350 2014-02-23 00:46:05Z jhb $");
| 30__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 266339 2014-05-17 19:11:08Z jhb $");
|
31
32#include <sys/param.h>
33#include <sys/lock.h>
34#include <sys/kernel.h>
35#include <sys/malloc.h>
36#include <sys/mutex.h>
37#include <sys/systm.h>
38#include <sys/smp.h>
39
| 31
32#include <sys/param.h>
33#include <sys/lock.h>
34#include <sys/kernel.h>
35#include <sys/malloc.h>
36#include <sys/mutex.h>
37#include <sys/systm.h>
38#include <sys/smp.h>
39
|
40#include <machine/clock.h>
| |
41#include <x86/specialreg.h>
42#include <x86/apicreg.h>
43
| 40#include <x86/specialreg.h>
41#include <x86/apicreg.h>
42
|
| 43#include <machine/clock.h>
44#include <machine/smp.h>
45
|
44#include <machine/vmm.h>
45
| 46#include <machine/vmm.h>
47
|
46#include "vmm_stat.h"
| 48#include "vmm_ipi.h"
|
47#include "vmm_lapic.h"
48#include "vmm_ktr.h"
| 49#include "vmm_lapic.h"
50#include "vmm_ktr.h"
|
| 51#include "vmm_stat.h"
52
|
49#include "vlapic.h"
| 53#include "vlapic.h"
|
| 54#include "vlapic_priv.h"
|
50#include "vioapic.h"
51
| 55#include "vioapic.h"
56
|
52#define VLAPIC_CTR0(vlapic, format) \
53 VCPU_CTR0((vlapic)->vm, (vlapic)->vcpuid, format)
54
55#define VLAPIC_CTR1(vlapic, format, p1) \
56 VCPU_CTR1((vlapic)->vm, (vlapic)->vcpuid, format, p1)
57
58#define VLAPIC_CTR2(vlapic, format, p1, p2) \
59 VCPU_CTR2((vlapic)->vm, (vlapic)->vcpuid, format, p1, p2)
60
61#define VLAPIC_CTR_IRR(vlapic, msg) \
62do { \
63 uint32_t *irrptr = &(vlapic)->apic.irr0; \
64 irrptr[0] = irrptr[0]; /* silence compiler */ \
65 VLAPIC_CTR1((vlapic), msg " irr0 0x%08x", irrptr[0 << 2]); \
66 VLAPIC_CTR1((vlapic), msg " irr1 0x%08x", irrptr[1 << 2]); \
67 VLAPIC_CTR1((vlapic), msg " irr2 0x%08x", irrptr[2 << 2]); \
68 VLAPIC_CTR1((vlapic), msg " irr3 0x%08x", irrptr[3 << 2]); \
69 VLAPIC_CTR1((vlapic), msg " irr4 0x%08x", irrptr[4 << 2]); \
70 VLAPIC_CTR1((vlapic), msg " irr5 0x%08x", irrptr[5 << 2]); \
71 VLAPIC_CTR1((vlapic), msg " irr6 0x%08x", irrptr[6 << 2]); \
72 VLAPIC_CTR1((vlapic), msg " irr7 0x%08x", irrptr[7 << 2]); \
73} while (0)
74
75#define VLAPIC_CTR_ISR(vlapic, msg) \
76do { \
77 uint32_t *isrptr = &(vlapic)->apic.isr0; \
78 isrptr[0] = isrptr[0]; /* silence compiler */ \
79 VLAPIC_CTR1((vlapic), msg " isr0 0x%08x", isrptr[0 << 2]); \
80 VLAPIC_CTR1((vlapic), msg " isr1 0x%08x", isrptr[1 << 2]); \
81 VLAPIC_CTR1((vlapic), msg " isr2 0x%08x", isrptr[2 << 2]); \
82 VLAPIC_CTR1((vlapic), msg " isr3 0x%08x", isrptr[3 << 2]); \
83 VLAPIC_CTR1((vlapic), msg " isr4 0x%08x", isrptr[4 << 2]); \
84 VLAPIC_CTR1((vlapic), msg " isr5 0x%08x", isrptr[5 << 2]); \
85 VLAPIC_CTR1((vlapic), msg " isr6 0x%08x", isrptr[6 << 2]); \
86 VLAPIC_CTR1((vlapic), msg " isr7 0x%08x", isrptr[7 << 2]); \
87} while (0)
88
89static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
90
| |
91#define PRIO(x) ((x) >> 4)
92
93#define VLAPIC_VERSION (16)
| 57#define PRIO(x) ((x) >> 4)
58
59#define VLAPIC_VERSION (16)
|
94#define VLAPIC_MAXLVT_ENTRIES (APIC_LVT_CMCI)
| |
95
96#define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
97
| 60
61#define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
62
|
98enum boot_state {
99 BS_INIT,
100 BS_SIPI,
101 BS_RUNNING
102};
103
104struct vlapic {
105 struct vm *vm;
106 int vcpuid;
107
108 struct LAPIC apic;
109
110 uint32_t esr_pending;
111 int esr_firing;
112
113 struct callout callout; /* vlapic timer */
114 struct bintime timer_fire_bt; /* callout expiry time */
115 struct bintime timer_freq_bt; /* timer frequency */
116 struct bintime timer_period_bt; /* timer period */
117 struct mtx timer_mtx;
118
119 /*
120 * The 'isrvec_stk' is a stack of vectors injected by the local apic.
121 * A vector is popped from the stack when the processor does an EOI.
122 * The vector on the top of the stack is used to compute the
123 * Processor Priority in conjunction with the TPR.
124 */
125 uint8_t isrvec_stk[ISRVEC_STK_SIZE];
126 int isrvec_stk_top;
127
128 uint64_t msr_apicbase;
129 enum boot_state boot_state;
130};
131
| |
132/*
133 * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
| 63/*
64 * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
|
134 * vlapic_callout_handler() and vcpu accesses to the following registers:
135 * - initial count register aka icr_timer
136 * - current count register aka ccr_timer
137 * - divide config register aka dcr_timer
| 65 * vlapic_callout_handler() and vcpu accesses to:
66 * - timer_freq_bt, timer_period_bt, timer_fire_bt
|
138 * - timer LVT register
| 67 * - timer LVT register
|
139 *
140 * Note that the vlapic_callout_handler() does not write to any of these
141 * registers so they can be safely read from the vcpu context without locking.
| |
142 */
143#define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx))
144#define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx))
145#define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx))
146
147#define VLAPIC_BUS_FREQ tsc_freq
148
149static __inline uint32_t
150vlapic_get_id(struct vlapic *vlapic)
151{
152
153 if (x2apic(vlapic))
154 return (vlapic->vcpuid);
155 else
156 return (vlapic->vcpuid << 24);
157}
158
| 68 */
69#define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx))
70#define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx))
71#define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx))
72
73#define VLAPIC_BUS_FREQ tsc_freq
74
75static __inline uint32_t
76vlapic_get_id(struct vlapic *vlapic)
77{
78
79 if (x2apic(vlapic))
80 return (vlapic->vcpuid);
81 else
82 return (vlapic->vcpuid << 24);
83}
84
|
159static __inline uint32_t
160vlapic_get_ldr(struct vlapic *vlapic)
| 85static uint32_t
86x2apic_ldr(struct vlapic *vlapic)
|
161{
| 87{
|
162 struct LAPIC *lapic;
| |
163 int apicid;
164 uint32_t ldr;
165
| 88 int apicid;
89 uint32_t ldr;
90
|
166 lapic = &vlapic->apic;
167 if (x2apic(vlapic)) {
168 apicid = vlapic_get_id(vlapic);
169 ldr = 1 << (apicid & 0xf);
170 ldr |= (apicid & 0xffff0) << 12;
171 return (ldr);
172 } else
173 return (lapic->ldr);
| 91 apicid = vlapic_get_id(vlapic);
92 ldr = 1 << (apicid & 0xf);
93 ldr |= (apicid & 0xffff0) << 12;
94 return (ldr);
|
174}
175
| 95}
96
|
176static __inline uint32_t
177vlapic_get_dfr(struct vlapic *vlapic)
| 97void
98vlapic_dfr_write_handler(struct vlapic *vlapic)
|
178{
179 struct LAPIC *lapic;
180
| 99{
100 struct LAPIC *lapic;
101
|
181 lapic = &vlapic->apic;
182 if (x2apic(vlapic))
183 return (0);
184 else
185 return (lapic->dfr);
186}
187
188static void
189vlapic_set_dfr(struct vlapic *vlapic, uint32_t data)
190{
191 uint32_t dfr;
192 struct LAPIC *lapic;
193
| 102 lapic = vlapic->apic_page;
|
194 if (x2apic(vlapic)) {
| 103 if (x2apic(vlapic)) {
|
195 VM_CTR1(vlapic->vm, "write to DFR in x2apic mode: %#x", data);
| 104 VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
105 lapic->dfr);
106 lapic->dfr = 0;
|
196 return;
197 }
198
| 107 return;
108 }
109
|
199 lapic = &vlapic->apic;
200 dfr = (lapic->dfr & APIC_DFR_RESERVED) | (data & APIC_DFR_MODEL_MASK);
201 if ((dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
| 110 lapic->dfr &= APIC_DFR_MODEL_MASK;
111 lapic->dfr |= APIC_DFR_RESERVED;
112
113 if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
|
202 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
| 114 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
|
203 else if ((dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
| 115 else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
|
204 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
205 else
| 116 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
117 else
|
206 VLAPIC_CTR1(vlapic, "vlapic DFR in Unknown Model %#x", dfr);
207
208 lapic->dfr = dfr;
| 118 VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
|
209}
210
| 119}
120
|
211static void
212vlapic_set_ldr(struct vlapic *vlapic, uint32_t data)
| 121void
122vlapic_ldr_write_handler(struct vlapic *vlapic)
|
213{
214 struct LAPIC *lapic;
215
| 123{
124 struct LAPIC *lapic;
125
|
| 126 lapic = vlapic->apic_page;
127
|
216 /* LDR is read-only in x2apic mode */
217 if (x2apic(vlapic)) {
| 128 /* LDR is read-only in x2apic mode */
129 if (x2apic(vlapic)) {
|
218 VLAPIC_CTR1(vlapic, "write to LDR in x2apic mode: %#x", data);
219 return;
| 130 VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
131 lapic->ldr);
132 lapic->ldr = x2apic_ldr(vlapic);
133 } else {
134 lapic->ldr &= ~APIC_LDR_RESERVED;
135 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
|
220 }
| 136 }
|
| 137}
|
221
| 138
|
222 lapic = &vlapic->apic;
223 lapic->ldr = data & ~APIC_LDR_RESERVED;
224 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
| 139void
140vlapic_id_write_handler(struct vlapic *vlapic)
141{
142 struct LAPIC *lapic;
143
144 /*
145 * We don't allow the ID register to be modified so reset it back to
146 * its default value.
147 */
148 lapic = vlapic->apic_page;
149 lapic->id = vlapic_get_id(vlapic);
|
225}
226
227static int
228vlapic_timer_divisor(uint32_t dcr)
229{
230 switch (dcr & 0xB) {
231 case APIC_TDCR_1:
232 return (1);
233 case APIC_TDCR_2:
234 return (2);
235 case APIC_TDCR_4:
236 return (4);
237 case APIC_TDCR_8:
238 return (8);
239 case APIC_TDCR_16:
240 return (16);
241 case APIC_TDCR_32:
242 return (32);
243 case APIC_TDCR_64:
244 return (64);
245 case APIC_TDCR_128:
246 return (128);
247 default:
248 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
249 }
250}
251
| 150}
151
152static int
153vlapic_timer_divisor(uint32_t dcr)
154{
155 switch (dcr & 0xB) {
156 case APIC_TDCR_1:
157 return (1);
158 case APIC_TDCR_2:
159 return (2);
160 case APIC_TDCR_4:
161 return (4);
162 case APIC_TDCR_8:
163 return (8);
164 case APIC_TDCR_16:
165 return (16);
166 case APIC_TDCR_32:
167 return (32);
168 case APIC_TDCR_64:
169 return (64);
170 case APIC_TDCR_128:
171 return (128);
172 default:
173 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
174 }
175}
176
|
252static void
253vlapic_mask_lvts(uint32_t *lvts, int num_lvt)
254{
255 int i;
256 for (i = 0; i < num_lvt; i++) {
257 *lvts |= APIC_LVT_M;
258 lvts += 4;
259 }
260}
261
| |
262#if 0
263static inline void
264vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
265{
266 printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
267 *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
268 *lvt & APIC_LVTT_M);
269}
270#endif
271
272static uint32_t
273vlapic_get_ccr(struct vlapic *vlapic)
274{
275 struct bintime bt_now, bt_rem;
276 struct LAPIC *lapic;
277 uint32_t ccr;
278
279 ccr = 0;
| 177#if 0
178static inline void
179vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
180{
181 printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
182 *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
183 *lvt & APIC_LVTT_M);
184}
185#endif
186
187static uint32_t
188vlapic_get_ccr(struct vlapic *vlapic)
189{
190 struct bintime bt_now, bt_rem;
191 struct LAPIC *lapic;
192 uint32_t ccr;
193
194 ccr = 0;
|
280 lapic = &vlapic->apic;
| 195 lapic = vlapic->apic_page;
|
281
282 VLAPIC_TIMER_LOCK(vlapic);
283 if (callout_active(&vlapic->callout)) {
284 /*
285 * If the timer is scheduled to expire in the future then
286 * compute the value of 'ccr' based on the remaining time.
287 */
288 binuptime(&bt_now);
289 if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
290 bt_rem = vlapic->timer_fire_bt;
291 bintime_sub(&bt_rem, &bt_now);
292 ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
293 ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
294 }
295 }
296 KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
297 "icr_timer is %#x", ccr, lapic->icr_timer));
298 VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
299 ccr, lapic->icr_timer);
300 VLAPIC_TIMER_UNLOCK(vlapic);
301 return (ccr);
302}
303
| 196
197 VLAPIC_TIMER_LOCK(vlapic);
198 if (callout_active(&vlapic->callout)) {
199 /*
200 * If the timer is scheduled to expire in the future then
201 * compute the value of 'ccr' based on the remaining time.
202 */
203 binuptime(&bt_now);
204 if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
205 bt_rem = vlapic->timer_fire_bt;
206 bintime_sub(&bt_rem, &bt_now);
207 ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
208 ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
209 }
210 }
211 KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
212 "icr_timer is %#x", ccr, lapic->icr_timer));
213 VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
214 ccr, lapic->icr_timer);
215 VLAPIC_TIMER_UNLOCK(vlapic);
216 return (ccr);
217}
218
|
304static void
305vlapic_set_dcr(struct vlapic *vlapic, uint32_t dcr)
| 219void
220vlapic_dcr_write_handler(struct vlapic *vlapic)
|
306{
307 struct LAPIC *lapic;
308 int divisor;
309
| 221{
222 struct LAPIC *lapic;
223 int divisor;
224
|
310 lapic = &vlapic->apic;
| 225 lapic = vlapic->apic_page;
|
311 VLAPIC_TIMER_LOCK(vlapic);
312
| 226 VLAPIC_TIMER_LOCK(vlapic);
227
|
313 lapic->dcr_timer = dcr;
314 divisor = vlapic_timer_divisor(dcr);
315 VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d", dcr, divisor);
| 228 divisor = vlapic_timer_divisor(lapic->dcr_timer);
229 VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
230 lapic->dcr_timer, divisor);
|
316
317 /*
318 * Update the timer frequency and the timer period.
319 *
320 * XXX changes to the frequency divider will not take effect until
321 * the timer is reloaded.
322 */
323 FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
324 vlapic->timer_period_bt = vlapic->timer_freq_bt;
325 bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
326
327 VLAPIC_TIMER_UNLOCK(vlapic);
328}
329
| 231
232 /*
233 * Update the timer frequency and the timer period.
234 *
235 * XXX changes to the frequency divider will not take effect until
236 * the timer is reloaded.
237 */
238 FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
239 vlapic->timer_period_bt = vlapic->timer_freq_bt;
240 bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
241
242 VLAPIC_TIMER_UNLOCK(vlapic);
243}
244
|
330static void
331vlapic_update_errors(struct vlapic *vlapic)
| 245void
246vlapic_esr_write_handler(struct vlapic *vlapic)
|
332{
| 247{
|
333 struct LAPIC *lapic = &vlapic->apic;
| 248 struct LAPIC *lapic;
249
250 lapic = vlapic->apic_page;
|
334 lapic->esr = vlapic->esr_pending;
335 vlapic->esr_pending = 0;
336}
337
| 251 lapic->esr = vlapic->esr_pending;
252 vlapic->esr_pending = 0;
253}
254
|
338static void
339vlapic_reset(struct vlapic *vlapic)
| 255int
256vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
|
340{
341 struct LAPIC *lapic;
| 257{
258 struct LAPIC *lapic;
|
342
343 lapic = &vlapic->apic;
344 bzero(lapic, sizeof(struct LAPIC));
| 259 uint32_t *irrptr, *tmrptr, mask;
260 int idx;
|
345
| 261
|
346 lapic->version = VLAPIC_VERSION;
347 lapic->version |= (VLAPIC_MAXLVT_ENTRIES << MAXLVTSHIFT);
348 lapic->dfr = 0xffffffff;
349 lapic->svr = APIC_SVR_VECTOR;
350 vlapic_mask_lvts(&lapic->lvt_timer, 6);
351 vlapic_mask_lvts(&lapic->lvt_cmci, 1);
352 vlapic_set_dcr(vlapic, 0);
| 262 KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
|
353
| 263
|
354 if (vlapic->vcpuid == 0)
355 vlapic->boot_state = BS_RUNNING; /* BSP */
356 else
357 vlapic->boot_state = BS_INIT; /* AP */
358}
359
360void
361vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
362{
363 struct LAPIC *lapic = &vlapic->apic;
364 uint32_t *irrptr, *tmrptr, mask;
365 int idx;
366
367 if (vector < 0 || vector >= 256)
368 panic("vlapic_set_intr_ready: invalid vector %d\n", vector);
369
| 264 lapic = vlapic->apic_page;
|
370 if (!(lapic->svr & APIC_SVR_ENABLE)) {
371 VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
372 "interrupt %d", vector);
| 265 if (!(lapic->svr & APIC_SVR_ENABLE)) {
266 VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
267 "interrupt %d", vector);
|
373 return;
| 268 return (0);
|
374 }
375
376 if (vector < 16) {
377 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
| 269 }
270
271 if (vector < 16) {
272 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
|
378 return;
| 273 VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
274 vector);
275 return (1);
|
379 }
| 276 }
|
380
| 277
278 if (vlapic->ops.set_intr_ready)
279 return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
280
|
381 idx = (vector / 32) * 4;
382 mask = 1 << (vector % 32);
383
384 irrptr = &lapic->irr0;
385 atomic_set_int(&irrptr[idx], mask);
386
387 /*
| 281 idx = (vector / 32) * 4;
282 mask = 1 << (vector % 32);
283
284 irrptr = &lapic->irr0;
285 atomic_set_int(&irrptr[idx], mask);
286
287 /*
|
388 * Upon acceptance of an interrupt into the IRR the corresponding
389 * TMR bit is cleared for edge-triggered interrupts and set for
390 * level-triggered interrupts.
| 288 * Verify that the trigger-mode of the interrupt matches with
289 * the vlapic TMR registers.
|
391 */
392 tmrptr = &lapic->tmr0;
| 290 */
291 tmrptr = &lapic->tmr0;
|
393 if (level)
394 atomic_set_int(&tmrptr[idx], mask);
395 else
396 atomic_clear_int(&tmrptr[idx], mask);
| 292 KASSERT((tmrptr[idx] & mask) == (level ? mask : 0),
293 ("vlapic TMR[%d] is 0x%08x but interrupt is %s-triggered",
294 idx / 4, tmrptr[idx], level ? "level" : "edge"));
|
397
398 VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
| 295
296 VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
|
| 297 return (1);
|
399}
400
401static __inline uint32_t *
402vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
403{
| 298}
299
300static __inline uint32_t *
301vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
302{
|
404 struct LAPIC *lapic = &vlapic->apic;
| 303 struct LAPIC *lapic = vlapic->apic_page;
|
405 int i;
406
407 switch (offset) {
408 case APIC_OFFSET_CMCI_LVT:
409 return (&lapic->lvt_cmci);
410 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
411 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
412 return ((&lapic->lvt_timer) + i);;
413 default:
414 panic("vlapic_get_lvt: invalid LVT\n");
415 }
416}
417
| 304 int i;
305
306 switch (offset) {
307 case APIC_OFFSET_CMCI_LVT:
308 return (&lapic->lvt_cmci);
309 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
310 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
311 return ((&lapic->lvt_timer) + i);;
312 default:
313 panic("vlapic_get_lvt: invalid LVT\n");
314 }
315}
316
|
| 317static __inline int
318lvt_off_to_idx(uint32_t offset)
319{
320 int index;
321
322 switch (offset) {
323 case APIC_OFFSET_CMCI_LVT:
324 index = APIC_LVT_CMCI;
325 break;
326 case APIC_OFFSET_TIMER_LVT:
327 index = APIC_LVT_TIMER;
328 break;
329 case APIC_OFFSET_THERM_LVT:
330 index = APIC_LVT_THERMAL;
331 break;
332 case APIC_OFFSET_PERF_LVT:
333 index = APIC_LVT_PMC;
334 break;
335 case APIC_OFFSET_LINT0_LVT:
336 index = APIC_LVT_LINT0;
337 break;
338 case APIC_OFFSET_LINT1_LVT:
339 index = APIC_LVT_LINT1;
340 break;
341 case APIC_OFFSET_ERROR_LVT:
342 index = APIC_LVT_ERROR;
343 break;
344 default:
345 index = -1;
346 break;
347 }
348 KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
349 "invalid lvt index %d for offset %#x", index, offset));
350
351 return (index);
352}
353
|
418static __inline uint32_t
419vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
420{
| 354static __inline uint32_t
355vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
356{
|
| 357 int idx;
358 uint32_t val;
|
421
| 359
|
422 return (*vlapic_get_lvtptr(vlapic, offset));
| 360 idx = lvt_off_to_idx(offset);
361 val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
362 return (val);
|
423}
424
| 363}
364
|
425static void
426vlapic_set_lvt(struct vlapic *vlapic, uint32_t offset, uint32_t val)
| 365void
366vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
|
427{
| 367{
|
428 uint32_t *lvtptr, mask;
| 368 uint32_t *lvtptr, mask, val;
|
429 struct LAPIC *lapic;
| 369 struct LAPIC *lapic;
|
| 370 int idx;
|
430
| 371
|
431 lapic = &vlapic->apic;
| 372 lapic = vlapic->apic_page;
|
432 lvtptr = vlapic_get_lvtptr(vlapic, offset);
| 373 lvtptr = vlapic_get_lvtptr(vlapic, offset);
|
| 374 val = *lvtptr;
375 idx = lvt_off_to_idx(offset);
|
433
| 376
|
434 if (offset == APIC_OFFSET_TIMER_LVT)
435 VLAPIC_TIMER_LOCK(vlapic);
436
| |
437 if (!(lapic->svr & APIC_SVR_ENABLE))
438 val |= APIC_LVT_M;
439 mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
440 switch (offset) {
441 case APIC_OFFSET_TIMER_LVT:
442 mask |= APIC_LVTT_TM;
443 break;
444 case APIC_OFFSET_ERROR_LVT:
445 break;
446 case APIC_OFFSET_LINT0_LVT:
447 case APIC_OFFSET_LINT1_LVT:
448 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
449 /* FALLTHROUGH */
450 default:
451 mask |= APIC_LVT_DM;
452 break;
453 }
| 377 if (!(lapic->svr & APIC_SVR_ENABLE))
378 val |= APIC_LVT_M;
379 mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
380 switch (offset) {
381 case APIC_OFFSET_TIMER_LVT:
382 mask |= APIC_LVTT_TM;
383 break;
384 case APIC_OFFSET_ERROR_LVT:
385 break;
386 case APIC_OFFSET_LINT0_LVT:
387 case APIC_OFFSET_LINT1_LVT:
388 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
389 /* FALLTHROUGH */
390 default:
391 mask |= APIC_LVT_DM;
392 break;
393 }
|
454 *lvtptr = val & mask;
| 394 val &= mask;
395 *lvtptr = val;
396 atomic_store_rel_32(&vlapic->lvt_last[idx], val);
397}
|
455
| 398
|
456 if (offset == APIC_OFFSET_TIMER_LVT)
457 VLAPIC_TIMER_UNLOCK(vlapic);
| 399static void
400vlapic_mask_lvts(struct vlapic *vlapic)
401{
402 struct LAPIC *lapic = vlapic->apic_page;
403
404 lapic->lvt_cmci |= APIC_LVT_M;
405 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
406
407 lapic->lvt_timer |= APIC_LVT_M;
408 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
409
410 lapic->lvt_thermal |= APIC_LVT_M;
411 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
412
413 lapic->lvt_pcint |= APIC_LVT_M;
414 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
415
416 lapic->lvt_lint0 |= APIC_LVT_M;
417 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
418
419 lapic->lvt_lint1 |= APIC_LVT_M;
420 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
421
422 lapic->lvt_error |= APIC_LVT_M;
423 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
|
458}
459
460static int
461vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
462{
463 uint32_t vec, mode;
464
465 if (lvt & APIC_LVT_M)
466 return (0);
467
468 vec = lvt & APIC_LVT_VECTOR;
469 mode = lvt & APIC_LVT_DM;
470
471 switch (mode) {
472 case APIC_LVT_DM_FIXED:
473 if (vec < 16) {
474 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
475 return (0);
476 }
| 424}
425
426static int
427vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
428{
429 uint32_t vec, mode;
430
431 if (lvt & APIC_LVT_M)
432 return (0);
433
434 vec = lvt & APIC_LVT_VECTOR;
435 mode = lvt & APIC_LVT_DM;
436
437 switch (mode) {
438 case APIC_LVT_DM_FIXED:
439 if (vec < 16) {
440 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
441 return (0);
442 }
|
477 vlapic_set_intr_ready(vlapic, vec, false);
478 vcpu_notify_event(vlapic->vm, vlapic->vcpuid);
| 443 if (vlapic_set_intr_ready(vlapic, vec, false))
444 vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
|
479 break;
480 case APIC_LVT_DM_NMI:
481 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
482 break;
483 default:
484 // Other modes ignored
485 return (0);
486 }
487 return (1);
488}
489
490#if 1
491static void
492dump_isrvec_stk(struct vlapic *vlapic)
493{
494 int i;
495 uint32_t *isrptr;
496
| 445 break;
446 case APIC_LVT_DM_NMI:
447 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
448 break;
449 default:
450 // Other modes ignored
451 return (0);
452 }
453 return (1);
454}
455
456#if 1
457static void
458dump_isrvec_stk(struct vlapic *vlapic)
459{
460 int i;
461 uint32_t *isrptr;
462
|
497 isrptr = &vlapic->apic.isr0;
| 463 isrptr = &vlapic->apic_page->isr0;
|
498 for (i = 0; i < 8; i++)
499 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
500
501 for (i = 0; i <= vlapic->isrvec_stk_top; i++)
502 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
503}
504#endif
505
506/*
507 * Algorithm adopted from section "Interrupt, Task and Processor Priority"
508 * in Intel Architecture Manual Vol 3a.
509 */
510static void
511vlapic_update_ppr(struct vlapic *vlapic)
512{
513 int isrvec, tpr, ppr;
514
515 /*
516 * Note that the value on the stack at index 0 is always 0.
517 *
518 * This is a placeholder for the value of ISRV when none of the
519 * bits is set in the ISRx registers.
520 */
521 isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
| 464 for (i = 0; i < 8; i++)
465 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
466
467 for (i = 0; i <= vlapic->isrvec_stk_top; i++)
468 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
469}
470#endif
471
472/*
473 * Algorithm adopted from section "Interrupt, Task and Processor Priority"
474 * in Intel Architecture Manual Vol 3a.
475 */
476static void
477vlapic_update_ppr(struct vlapic *vlapic)
478{
479 int isrvec, tpr, ppr;
480
481 /*
482 * Note that the value on the stack at index 0 is always 0.
483 *
484 * This is a placeholder for the value of ISRV when none of the
485 * bits is set in the ISRx registers.
486 */
487 isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
|
522 tpr = vlapic->apic.tpr;
| 488 tpr = vlapic->apic_page->tpr;
|
523
524#if 1
525 {
526 int i, lastprio, curprio, vector, idx;
527 uint32_t *isrptr;
528
529 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
530 panic("isrvec_stk is corrupted: %d", isrvec);
531
532 /*
533 * Make sure that the priority of the nested interrupts is
534 * always increasing.
535 */
536 lastprio = -1;
537 for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
538 curprio = PRIO(vlapic->isrvec_stk[i]);
539 if (curprio <= lastprio) {
540 dump_isrvec_stk(vlapic);
541 panic("isrvec_stk does not satisfy invariant");
542 }
543 lastprio = curprio;
544 }
545
546 /*
547 * Make sure that each bit set in the ISRx registers has a
548 * corresponding entry on the isrvec stack.
549 */
550 i = 1;
| 489
490#if 1
491 {
492 int i, lastprio, curprio, vector, idx;
493 uint32_t *isrptr;
494
495 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
496 panic("isrvec_stk is corrupted: %d", isrvec);
497
498 /*
499 * Make sure that the priority of the nested interrupts is
500 * always increasing.
501 */
502 lastprio = -1;
503 for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
504 curprio = PRIO(vlapic->isrvec_stk[i]);
505 if (curprio <= lastprio) {
506 dump_isrvec_stk(vlapic);
507 panic("isrvec_stk does not satisfy invariant");
508 }
509 lastprio = curprio;
510 }
511
512 /*
513 * Make sure that each bit set in the ISRx registers has a
514 * corresponding entry on the isrvec stack.
515 */
516 i = 1;
|
551 isrptr = &vlapic->apic.isr0;
| 517 isrptr = &vlapic->apic_page->isr0;
|
552 for (vector = 0; vector < 256; vector++) {
553 idx = (vector / 32) * 4;
554 if (isrptr[idx] & (1 << (vector % 32))) {
555 if (i > vlapic->isrvec_stk_top ||
556 vlapic->isrvec_stk[i] != vector) {
557 dump_isrvec_stk(vlapic);
558 panic("ISR and isrvec_stk out of sync");
559 }
560 i++;
561 }
562 }
563 }
564#endif
565
566 if (PRIO(tpr) >= PRIO(isrvec))
567 ppr = tpr;
568 else
569 ppr = isrvec & 0xf0;
570
| 518 for (vector = 0; vector < 256; vector++) {
519 idx = (vector / 32) * 4;
520 if (isrptr[idx] & (1 << (vector % 32))) {
521 if (i > vlapic->isrvec_stk_top ||
522 vlapic->isrvec_stk[i] != vector) {
523 dump_isrvec_stk(vlapic);
524 panic("ISR and isrvec_stk out of sync");
525 }
526 i++;
527 }
528 }
529 }
530#endif
531
532 if (PRIO(tpr) >= PRIO(isrvec))
533 ppr = tpr;
534 else
535 ppr = isrvec & 0xf0;
536
|
571 vlapic->apic.ppr = ppr;
| 537 vlapic->apic_page->ppr = ppr;
|
572 VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
573}
574
575static void
576vlapic_process_eoi(struct vlapic *vlapic)
577{
| 538 VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
539}
540
541static void
542vlapic_process_eoi(struct vlapic *vlapic)
543{
|
578 struct LAPIC *lapic = &vlapic->apic;
| 544 struct LAPIC *lapic = vlapic->apic_page;
|
579 uint32_t *isrptr, *tmrptr;
580 int i, idx, bitpos, vector;
581
582 isrptr = &lapic->isr0;
583 tmrptr = &lapic->tmr0;
584
585 /*
586 * The x86 architecture reserves the the first 32 vectors for use
587 * by the processor.
588 */
589 for (i = 7; i > 0; i--) {
590 idx = i * 4;
591 bitpos = fls(isrptr[idx]);
592 if (bitpos-- != 0) {
593 if (vlapic->isrvec_stk_top <= 0) {
594 panic("invalid vlapic isrvec_stk_top %d",
595 vlapic->isrvec_stk_top);
596 }
597 isrptr[idx] &= ~(1 << bitpos);
598 VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
599 vlapic->isrvec_stk_top--;
600 vlapic_update_ppr(vlapic);
601 if ((tmrptr[idx] & (1 << bitpos)) != 0) {
602 vector = i * 32 + bitpos;
603 vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
604 vector);
605 }
606 return;
607 }
608 }
609}
610
611static __inline int
612vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
613{
614
615 return (lvt & mask);
616}
617
618static __inline int
619vlapic_periodic_timer(struct vlapic *vlapic)
620{
621 uint32_t lvt;
622
623 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
624
625 return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
626}
627
628static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
629
630void
631vlapic_set_error(struct vlapic *vlapic, uint32_t mask)
632{
633 uint32_t lvt;
634
635 vlapic->esr_pending |= mask;
636 if (vlapic->esr_firing)
637 return;
638 vlapic->esr_firing = 1;
639
640 // The error LVT always uses the fixed delivery mode.
641 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
642 if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
643 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1);
644 }
645 vlapic->esr_firing = 0;
646}
647
648static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
649
650static void
651vlapic_fire_timer(struct vlapic *vlapic)
652{
653 uint32_t lvt;
654
655 KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
656
657 // The timer LVT always uses the fixed delivery mode.
658 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
659 if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
660 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
661 }
662}
663
664static VMM_STAT(VLAPIC_INTR_CMC,
665 "corrected machine check interrupts generated by vlapic");
666
667void
668vlapic_fire_cmci(struct vlapic *vlapic)
669{
670 uint32_t lvt;
671
672 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
673 if (vlapic_fire_lvt(vlapic, lvt)) {
674 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
675 }
676}
677
| 545 uint32_t *isrptr, *tmrptr;
546 int i, idx, bitpos, vector;
547
548 isrptr = &lapic->isr0;
549 tmrptr = &lapic->tmr0;
550
551 /*
552 * The x86 architecture reserves the the first 32 vectors for use
553 * by the processor.
554 */
555 for (i = 7; i > 0; i--) {
556 idx = i * 4;
557 bitpos = fls(isrptr[idx]);
558 if (bitpos-- != 0) {
559 if (vlapic->isrvec_stk_top <= 0) {
560 panic("invalid vlapic isrvec_stk_top %d",
561 vlapic->isrvec_stk_top);
562 }
563 isrptr[idx] &= ~(1 << bitpos);
564 VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
565 vlapic->isrvec_stk_top--;
566 vlapic_update_ppr(vlapic);
567 if ((tmrptr[idx] & (1 << bitpos)) != 0) {
568 vector = i * 32 + bitpos;
569 vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
570 vector);
571 }
572 return;
573 }
574 }
575}
576
577static __inline int
578vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
579{
580
581 return (lvt & mask);
582}
583
584static __inline int
585vlapic_periodic_timer(struct vlapic *vlapic)
586{
587 uint32_t lvt;
588
589 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
590
591 return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
592}
593
594static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
595
596void
597vlapic_set_error(struct vlapic *vlapic, uint32_t mask)
598{
599 uint32_t lvt;
600
601 vlapic->esr_pending |= mask;
602 if (vlapic->esr_firing)
603 return;
604 vlapic->esr_firing = 1;
605
606 // The error LVT always uses the fixed delivery mode.
607 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
608 if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
609 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1);
610 }
611 vlapic->esr_firing = 0;
612}
613
614static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
615
616static void
617vlapic_fire_timer(struct vlapic *vlapic)
618{
619 uint32_t lvt;
620
621 KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
622
623 // The timer LVT always uses the fixed delivery mode.
624 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
625 if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
626 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
627 }
628}
629
630static VMM_STAT(VLAPIC_INTR_CMC,
631 "corrected machine check interrupts generated by vlapic");
632
633void
634vlapic_fire_cmci(struct vlapic *vlapic)
635{
636 uint32_t lvt;
637
638 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
639 if (vlapic_fire_lvt(vlapic, lvt)) {
640 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
641 }
642}
643
|
678static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_ENTRIES,
| 644static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
|
679 "lvts triggered");
680
681int
682vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
683{
684 uint32_t lvt;
685
686 switch (vector) {
687 case APIC_LVT_LINT0:
688 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT);
689 break;
690 case APIC_LVT_LINT1:
691 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT);
692 break;
693 case APIC_LVT_TIMER:
694 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
695 lvt |= APIC_LVT_DM_FIXED;
696 break;
697 case APIC_LVT_ERROR:
698 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
699 lvt |= APIC_LVT_DM_FIXED;
700 break;
701 case APIC_LVT_PMC:
702 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT);
703 break;
704 case APIC_LVT_THERMAL:
705 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT);
706 break;
707 case APIC_LVT_CMCI:
708 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
709 break;
710 default:
711 return (EINVAL);
712 }
713 if (vlapic_fire_lvt(vlapic, lvt)) {
714 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
715 LVTS_TRIGGERRED, vector, 1);
716 }
717 return (0);
718}
719
720static void
721vlapic_callout_handler(void *arg)
722{
723 struct vlapic *vlapic;
724 struct bintime bt, btnow;
725 sbintime_t rem_sbt;
726
727 vlapic = arg;
728
729 VLAPIC_TIMER_LOCK(vlapic);
730 if (callout_pending(&vlapic->callout)) /* callout was reset */
731 goto done;
732
733 if (!callout_active(&vlapic->callout)) /* callout was stopped */
734 goto done;
735
736 callout_deactivate(&vlapic->callout);
737
| 645 "lvts triggered");
646
647int
648vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
649{
650 uint32_t lvt;
651
652 switch (vector) {
653 case APIC_LVT_LINT0:
654 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT);
655 break;
656 case APIC_LVT_LINT1:
657 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT);
658 break;
659 case APIC_LVT_TIMER:
660 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
661 lvt |= APIC_LVT_DM_FIXED;
662 break;
663 case APIC_LVT_ERROR:
664 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
665 lvt |= APIC_LVT_DM_FIXED;
666 break;
667 case APIC_LVT_PMC:
668 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT);
669 break;
670 case APIC_LVT_THERMAL:
671 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT);
672 break;
673 case APIC_LVT_CMCI:
674 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
675 break;
676 default:
677 return (EINVAL);
678 }
679 if (vlapic_fire_lvt(vlapic, lvt)) {
680 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
681 LVTS_TRIGGERRED, vector, 1);
682 }
683 return (0);
684}
685
686static void
687vlapic_callout_handler(void *arg)
688{
689 struct vlapic *vlapic;
690 struct bintime bt, btnow;
691 sbintime_t rem_sbt;
692
693 vlapic = arg;
694
695 VLAPIC_TIMER_LOCK(vlapic);
696 if (callout_pending(&vlapic->callout)) /* callout was reset */
697 goto done;
698
699 if (!callout_active(&vlapic->callout)) /* callout was stopped */
700 goto done;
701
702 callout_deactivate(&vlapic->callout);
703
|
738 KASSERT(vlapic->apic.icr_timer != 0, ("vlapic timer is disabled"));
739
| |
740 vlapic_fire_timer(vlapic);
741
742 if (vlapic_periodic_timer(vlapic)) {
743 binuptime(&btnow);
744 KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
745 ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
746 btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
747 vlapic->timer_fire_bt.frac));
748
749 /*
750 * Compute the delta between when the timer was supposed to
751 * fire and the present time.
752 */
753 bt = btnow;
754 bintime_sub(&bt, &vlapic->timer_fire_bt);
755
756 rem_sbt = bttosbt(vlapic->timer_period_bt);
757 if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
758 /*
759 * Adjust the time until the next countdown downward
760 * to account for the lost time.
761 */
762 rem_sbt -= bttosbt(bt);
763 } else {
764 /*
765 * If the delta is greater than the timer period then
766 * just reset our time base instead of trying to catch
767 * up.
768 */
769 vlapic->timer_fire_bt = btnow;
770 VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
771 "usecs, period is %lu usecs - resetting time base",
772 bttosbt(bt) / SBT_1US,
773 bttosbt(vlapic->timer_period_bt) / SBT_1US);
774 }
775
776 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
777 callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
778 vlapic_callout_handler, vlapic, 0);
779 }
780done:
781 VLAPIC_TIMER_UNLOCK(vlapic);
782}
783
| 704 vlapic_fire_timer(vlapic);
705
706 if (vlapic_periodic_timer(vlapic)) {
707 binuptime(&btnow);
708 KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
709 ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
710 btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
711 vlapic->timer_fire_bt.frac));
712
713 /*
714 * Compute the delta between when the timer was supposed to
715 * fire and the present time.
716 */
717 bt = btnow;
718 bintime_sub(&bt, &vlapic->timer_fire_bt);
719
720 rem_sbt = bttosbt(vlapic->timer_period_bt);
721 if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
722 /*
723 * Adjust the time until the next countdown downward
724 * to account for the lost time.
725 */
726 rem_sbt -= bttosbt(bt);
727 } else {
728 /*
729 * If the delta is greater than the timer period then
730 * just reset our time base instead of trying to catch
731 * up.
732 */
733 vlapic->timer_fire_bt = btnow;
734 VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
735 "usecs, period is %lu usecs - resetting time base",
736 bttosbt(bt) / SBT_1US,
737 bttosbt(vlapic->timer_period_bt) / SBT_1US);
738 }
739
740 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
741 callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
742 vlapic_callout_handler, vlapic, 0);
743 }
744done:
745 VLAPIC_TIMER_UNLOCK(vlapic);
746}
747
|
784static void
785vlapic_set_icr_timer(struct vlapic *vlapic, uint32_t icr_timer)
| 748void
749vlapic_icrtmr_write_handler(struct vlapic *vlapic)
|
786{
787 struct LAPIC *lapic;
788 sbintime_t sbt;
| 750{
751 struct LAPIC *lapic;
752 sbintime_t sbt;
|
| 753 uint32_t icr_timer;
|
789
790 VLAPIC_TIMER_LOCK(vlapic);
791
| 754
755 VLAPIC_TIMER_LOCK(vlapic);
756
|
792 lapic = &vlapic->apic;
793 lapic->icr_timer = icr_timer;
| 757 lapic = vlapic->apic_page;
758 icr_timer = lapic->icr_timer;
|
794
795 vlapic->timer_period_bt = vlapic->timer_freq_bt;
796 bintime_mul(&vlapic->timer_period_bt, icr_timer);
797
798 if (icr_timer != 0) {
799 binuptime(&vlapic->timer_fire_bt);
800 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
801
802 sbt = bttosbt(vlapic->timer_period_bt);
803 callout_reset_sbt(&vlapic->callout, sbt, 0,
804 vlapic_callout_handler, vlapic, 0);
805 } else
806 callout_stop(&vlapic->callout);
807
808 VLAPIC_TIMER_UNLOCK(vlapic);
809}
810
811/*
812 * This function populates 'dmask' with the set of vcpus that match the
813 * addressing specified by the (dest, phys, lowprio) tuple.
814 *
815 * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
816 * or xAPIC (8-bit) destination field.
817 */
818static void
819vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
820 bool lowprio, bool x2apic_dest)
821{
822 struct vlapic *vlapic;
823 uint32_t dfr, ldr, ldest, cluster;
824 uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
825 cpuset_t amask;
826 int vcpuid;
827
828 if ((x2apic_dest && dest == 0xffffffff) ||
829 (!x2apic_dest && dest == 0xff)) {
830 /*
831 * Broadcast in both logical and physical modes.
832 */
833 *dmask = vm_active_cpus(vm);
834 return;
835 }
836
837 if (phys) {
838 /*
839 * Physical mode: destination is APIC ID.
840 */
841 CPU_ZERO(dmask);
842 vcpuid = vm_apicid2vcpuid(vm, dest);
843 if (vcpuid < VM_MAXCPU)
844 CPU_SET(vcpuid, dmask);
845 } else {
846 /*
847 * In the "Flat Model" the MDA is interpreted as an 8-bit wide
848 * bitmask. This model is only avilable in the xAPIC mode.
849 */
850 mda_flat_ldest = dest & 0xff;
851
852 /*
853 * In the "Cluster Model" the MDA is used to identify a
854 * specific cluster and a set of APICs in that cluster.
855 */
856 if (x2apic_dest) {
857 mda_cluster_id = dest >> 16;
858 mda_cluster_ldest = dest & 0xffff;
859 } else {
860 mda_cluster_id = (dest >> 4) & 0xf;
861 mda_cluster_ldest = dest & 0xf;
862 }
863
864 /*
865 * Logical mode: match each APIC that has a bit set
866 * in it's LDR that matches a bit in the ldest.
867 */
868 CPU_ZERO(dmask);
869 amask = vm_active_cpus(vm);
870 while ((vcpuid = CPU_FFS(&amask)) != 0) {
871 vcpuid--;
872 CPU_CLR(vcpuid, &amask);
873
874 vlapic = vm_lapic(vm, vcpuid);
| 759
760 vlapic->timer_period_bt = vlapic->timer_freq_bt;
761 bintime_mul(&vlapic->timer_period_bt, icr_timer);
762
763 if (icr_timer != 0) {
764 binuptime(&vlapic->timer_fire_bt);
765 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
766
767 sbt = bttosbt(vlapic->timer_period_bt);
768 callout_reset_sbt(&vlapic->callout, sbt, 0,
769 vlapic_callout_handler, vlapic, 0);
770 } else
771 callout_stop(&vlapic->callout);
772
773 VLAPIC_TIMER_UNLOCK(vlapic);
774}
775
776/*
777 * This function populates 'dmask' with the set of vcpus that match the
778 * addressing specified by the (dest, phys, lowprio) tuple.
779 *
780 * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
781 * or xAPIC (8-bit) destination field.
782 */
783static void
784vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
785 bool lowprio, bool x2apic_dest)
786{
787 struct vlapic *vlapic;
788 uint32_t dfr, ldr, ldest, cluster;
789 uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
790 cpuset_t amask;
791 int vcpuid;
792
793 if ((x2apic_dest && dest == 0xffffffff) ||
794 (!x2apic_dest && dest == 0xff)) {
795 /*
796 * Broadcast in both logical and physical modes.
797 */
798 *dmask = vm_active_cpus(vm);
799 return;
800 }
801
802 if (phys) {
803 /*
804 * Physical mode: destination is APIC ID.
805 */
806 CPU_ZERO(dmask);
807 vcpuid = vm_apicid2vcpuid(vm, dest);
808 if (vcpuid < VM_MAXCPU)
809 CPU_SET(vcpuid, dmask);
810 } else {
811 /*
812 * In the "Flat Model" the MDA is interpreted as an 8-bit wide
813 * bitmask. This model is only avilable in the xAPIC mode.
814 */
815 mda_flat_ldest = dest & 0xff;
816
817 /*
818 * In the "Cluster Model" the MDA is used to identify a
819 * specific cluster and a set of APICs in that cluster.
820 */
821 if (x2apic_dest) {
822 mda_cluster_id = dest >> 16;
823 mda_cluster_ldest = dest & 0xffff;
824 } else {
825 mda_cluster_id = (dest >> 4) & 0xf;
826 mda_cluster_ldest = dest & 0xf;
827 }
828
829 /*
830 * Logical mode: match each APIC that has a bit set
831 * in it's LDR that matches a bit in the ldest.
832 */
833 CPU_ZERO(dmask);
834 amask = vm_active_cpus(vm);
835 while ((vcpuid = CPU_FFS(&amask)) != 0) {
836 vcpuid--;
837 CPU_CLR(vcpuid, &amask);
838
839 vlapic = vm_lapic(vm, vcpuid);
|
875 dfr = vlapic_get_dfr(vlapic);
876 ldr = vlapic_get_ldr(vlapic);
| 840 dfr = vlapic->apic_page->dfr;
841 ldr = vlapic->apic_page->ldr;
|
877
878 if ((dfr & APIC_DFR_MODEL_MASK) ==
879 APIC_DFR_MODEL_FLAT) {
880 ldest = ldr >> 24;
881 mda_ldest = mda_flat_ldest;
882 } else if ((dfr & APIC_DFR_MODEL_MASK) ==
883 APIC_DFR_MODEL_CLUSTER) {
884 if (x2apic(vlapic)) {
885 cluster = ldr >> 16;
886 ldest = ldr & 0xffff;
887 } else {
888 cluster = ldr >> 28;
889 ldest = (ldr >> 24) & 0xf;
890 }
891 if (cluster != mda_cluster_id)
892 continue;
893 mda_ldest = mda_cluster_ldest;
894 } else {
895 /*
896 * Guest has configured a bad logical
897 * model for this vcpu - skip it.
898 */
899 VLAPIC_CTR1(vlapic, "vlapic has bad logical "
900 "model %x - cannot deliver interrupt", dfr);
901 continue;
902 }
903
904 if ((mda_ldest & ldest) != 0) {
905 CPU_SET(vcpuid, dmask);
906 if (lowprio)
907 break;
908 }
909 }
910 }
911}
912
913static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
914
| 842
843 if ((dfr & APIC_DFR_MODEL_MASK) ==
844 APIC_DFR_MODEL_FLAT) {
845 ldest = ldr >> 24;
846 mda_ldest = mda_flat_ldest;
847 } else if ((dfr & APIC_DFR_MODEL_MASK) ==
848 APIC_DFR_MODEL_CLUSTER) {
849 if (x2apic(vlapic)) {
850 cluster = ldr >> 16;
851 ldest = ldr & 0xffff;
852 } else {
853 cluster = ldr >> 28;
854 ldest = (ldr >> 24) & 0xf;
855 }
856 if (cluster != mda_cluster_id)
857 continue;
858 mda_ldest = mda_cluster_ldest;
859 } else {
860 /*
861 * Guest has configured a bad logical
862 * model for this vcpu - skip it.
863 */
864 VLAPIC_CTR1(vlapic, "vlapic has bad logical "
865 "model %x - cannot deliver interrupt", dfr);
866 continue;
867 }
868
869 if ((mda_ldest & ldest) != 0) {
870 CPU_SET(vcpuid, dmask);
871 if (lowprio)
872 break;
873 }
874 }
875 }
876}
877
878static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
879
|
915static int
916lapic_process_icr(struct vlapic *vlapic, uint64_t icrval, bool *retu)
| 880int
881vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
|
917{
918 int i;
919 bool phys;
920 cpuset_t dmask;
| 882{
883 int i;
884 bool phys;
885 cpuset_t dmask;
|
| 886 uint64_t icrval;
|
921 uint32_t dest, vec, mode;
922 struct vlapic *vlapic2;
923 struct vm_exit *vmexit;
| 887 uint32_t dest, vec, mode;
888 struct vlapic *vlapic2;
889 struct vm_exit *vmexit;
|
924
| 890 struct LAPIC *lapic;
891
892 lapic = vlapic->apic_page;
893 lapic->icr_lo &= ~APIC_DELSTAT_PEND;
894 icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
895
|
925 if (x2apic(vlapic))
926 dest = icrval >> 32;
927 else
928 dest = icrval >> (32 + 24);
929 vec = icrval & APIC_VECTOR_MASK;
930 mode = icrval & APIC_DELMODE_MASK;
931
932 if (mode == APIC_DELMODE_FIXED && vec < 16) {
933 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
| 896 if (x2apic(vlapic))
897 dest = icrval >> 32;
898 else
899 dest = icrval >> (32 + 24);
900 vec = icrval & APIC_VECTOR_MASK;
901 mode = icrval & APIC_DELMODE_MASK;
902
903 if (mode == APIC_DELMODE_FIXED && vec < 16) {
904 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
|
| 905 VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
|
934 return (0);
935 }
| 906 return (0);
907 }
|
936
| 908
909 VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
910
|
937 if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
938 switch (icrval & APIC_DEST_MASK) {
939 case APIC_DEST_DESTFLD:
940 phys = ((icrval & APIC_DESTMODE_LOG) == 0);
941 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
942 x2apic(vlapic));
943 break;
944 case APIC_DEST_SELF:
945 CPU_SETOF(vlapic->vcpuid, &dmask);
946 break;
947 case APIC_DEST_ALLISELF:
948 dmask = vm_active_cpus(vlapic->vm);
949 break;
950 case APIC_DEST_ALLESELF:
951 dmask = vm_active_cpus(vlapic->vm);
952 CPU_CLR(vlapic->vcpuid, &dmask);
953 break;
954 default:
955 CPU_ZERO(&dmask); /* satisfy gcc */
956 break;
957 }
958
959 while ((i = CPU_FFS(&dmask)) != 0) {
960 i--;
961 CPU_CLR(i, &dmask);
962 if (mode == APIC_DELMODE_FIXED) {
963 lapic_intr_edge(vlapic->vm, i, vec);
964 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
965 IPIS_SENT, i, 1);
| 911 if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
912 switch (icrval & APIC_DEST_MASK) {
913 case APIC_DEST_DESTFLD:
914 phys = ((icrval & APIC_DESTMODE_LOG) == 0);
915 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
916 x2apic(vlapic));
917 break;
918 case APIC_DEST_SELF:
919 CPU_SETOF(vlapic->vcpuid, &dmask);
920 break;
921 case APIC_DEST_ALLISELF:
922 dmask = vm_active_cpus(vlapic->vm);
923 break;
924 case APIC_DEST_ALLESELF:
925 dmask = vm_active_cpus(vlapic->vm);
926 CPU_CLR(vlapic->vcpuid, &dmask);
927 break;
928 default:
929 CPU_ZERO(&dmask); /* satisfy gcc */
930 break;
931 }
932
933 while ((i = CPU_FFS(&dmask)) != 0) {
934 i--;
935 CPU_CLR(i, &dmask);
936 if (mode == APIC_DELMODE_FIXED) {
937 lapic_intr_edge(vlapic->vm, i, vec);
938 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
939 IPIS_SENT, i, 1);
|
966 } else
| 940 VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
941 "to vcpuid %d", vec, i);
942 } else {
|
967 vm_inject_nmi(vlapic->vm, i);
| 943 vm_inject_nmi(vlapic->vm, i);
|
| 944 VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
945 "to vcpuid %d", i);
946 }
|
968 }
969
970 return (0); /* handled completely in the kernel */
971 }
972
973 if (mode == APIC_DELMODE_INIT) {
974 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
975 return (0);
976
977 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
978 vlapic2 = vm_lapic(vlapic->vm, dest);
979
980 /* move from INIT to waiting-for-SIPI state */
981 if (vlapic2->boot_state == BS_INIT) {
982 vlapic2->boot_state = BS_SIPI;
983 }
984
985 return (0);
986 }
987 }
988
989 if (mode == APIC_DELMODE_STARTUP) {
990 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
991 vlapic2 = vm_lapic(vlapic->vm, dest);
992
993 /*
994 * Ignore SIPIs in any state other than wait-for-SIPI
995 */
996 if (vlapic2->boot_state != BS_SIPI)
997 return (0);
998
999 /*
1000 * XXX this assumes that the startup IPI always succeeds
1001 */
1002 vlapic2->boot_state = BS_RUNNING;
1003 vm_activate_cpu(vlapic2->vm, dest);
1004
1005 *retu = true;
1006 vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
1007 vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
1008 vmexit->u.spinup_ap.vcpu = dest;
1009 vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
1010
1011 return (0);
1012 }
1013 }
1014
1015 /*
1016 * This will cause a return to userland.
1017 */
1018 return (1);
1019}
1020
1021int
| 947 }
948
949 return (0); /* handled completely in the kernel */
950 }
951
952 if (mode == APIC_DELMODE_INIT) {
953 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
954 return (0);
955
956 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
957 vlapic2 = vm_lapic(vlapic->vm, dest);
958
959 /* move from INIT to waiting-for-SIPI state */
960 if (vlapic2->boot_state == BS_INIT) {
961 vlapic2->boot_state = BS_SIPI;
962 }
963
964 return (0);
965 }
966 }
967
968 if (mode == APIC_DELMODE_STARTUP) {
969 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
970 vlapic2 = vm_lapic(vlapic->vm, dest);
971
972 /*
973 * Ignore SIPIs in any state other than wait-for-SIPI
974 */
975 if (vlapic2->boot_state != BS_SIPI)
976 return (0);
977
978 /*
979 * XXX this assumes that the startup IPI always succeeds
980 */
981 vlapic2->boot_state = BS_RUNNING;
982 vm_activate_cpu(vlapic2->vm, dest);
983
984 *retu = true;
985 vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
986 vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
987 vmexit->u.spinup_ap.vcpu = dest;
988 vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
989
990 return (0);
991 }
992 }
993
994 /*
995 * This will cause a return to userland.
996 */
997 return (1);
998}
999
1000int
|
1022vlapic_pending_intr(struct vlapic *vlapic)
| 1001vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
|
1023{
| 1002{
|
1024 struct LAPIC *lapic = &vlapic->apic;
| 1003 struct LAPIC *lapic = vlapic->apic_page;
|
1025 int idx, i, bitpos, vector;
1026 uint32_t *irrptr, val;
1027
| 1004 int idx, i, bitpos, vector;
1005 uint32_t *irrptr, val;
1006
|
| 1007 if (vlapic->ops.pending_intr)
1008 return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
1009
|
1028 irrptr = &lapic->irr0;
1029
1030 /*
1031 * The x86 architecture reserves the the first 32 vectors for use
1032 * by the processor.
1033 */
1034 for (i = 7; i > 0; i--) {
1035 idx = i * 4;
1036 val = atomic_load_acq_int(&irrptr[idx]);
1037 bitpos = fls(val);
1038 if (bitpos != 0) {
1039 vector = i * 32 + (bitpos - 1);
1040 if (PRIO(vector) > PRIO(lapic->ppr)) {
1041 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
| 1010 irrptr = &lapic->irr0;
1011
1012 /*
1013 * The x86 architecture reserves the the first 32 vectors for use
1014 * by the processor.
1015 */
1016 for (i = 7; i > 0; i--) {
1017 idx = i * 4;
1018 val = atomic_load_acq_int(&irrptr[idx]);
1019 bitpos = fls(val);
1020 if (bitpos != 0) {
1021 vector = i * 32 + (bitpos - 1);
1022 if (PRIO(vector) > PRIO(lapic->ppr)) {
1023 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
|
1042 return (vector);
| 1024 if (vecptr != NULL)
1025 *vecptr = vector;
1026 return (1);
|
1043 } else
1044 break;
1045 }
1046 }
| 1027 } else
1028 break;
1029 }
1030 }
|
1047 return (-1);
| 1031 return (0);
|
1048}
1049
1050void
1051vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1052{
| 1032}
1033
1034void
1035vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1036{
|
1053 struct LAPIC *lapic = &vlapic->apic;
| 1037 struct LAPIC *lapic = vlapic->apic_page;
|
1054 uint32_t *irrptr, *isrptr;
1055 int idx, stk_top;
1056
| 1038 uint32_t *irrptr, *isrptr;
1039 int idx, stk_top;
1040
|
| 1041 if (vlapic->ops.intr_accepted)
1042 return ((*vlapic->ops.intr_accepted)(vlapic, vector));
1043
|
1057 /*
1058 * clear the ready bit for vector being accepted in irr
1059 * and set the vector as in service in isr.
1060 */
1061 idx = (vector / 32) * 4;
1062
1063 irrptr = &lapic->irr0;
1064 atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
1065 VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
1066
1067 isrptr = &lapic->isr0;
1068 isrptr[idx] |= 1 << (vector % 32);
1069 VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
1070
1071 /*
1072 * Update the PPR
1073 */
1074 vlapic->isrvec_stk_top++;
1075
1076 stk_top = vlapic->isrvec_stk_top;
1077 if (stk_top >= ISRVEC_STK_SIZE)
1078 panic("isrvec_stk_top overflow %d", stk_top);
1079
1080 vlapic->isrvec_stk[stk_top] = vector;
1081 vlapic_update_ppr(vlapic);
1082}
1083
| 1044 /*
1045 * clear the ready bit for vector being accepted in irr
1046 * and set the vector as in service in isr.
1047 */
1048 idx = (vector / 32) * 4;
1049
1050 irrptr = &lapic->irr0;
1051 atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
1052 VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
1053
1054 isrptr = &lapic->isr0;
1055 isrptr[idx] |= 1 << (vector % 32);
1056 VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
1057
1058 /*
1059 * Update the PPR
1060 */
1061 vlapic->isrvec_stk_top++;
1062
1063 stk_top = vlapic->isrvec_stk_top;
1064 if (stk_top >= ISRVEC_STK_SIZE)
1065 panic("isrvec_stk_top overflow %d", stk_top);
1066
1067 vlapic->isrvec_stk[stk_top] = vector;
1068 vlapic_update_ppr(vlapic);
1069}
1070
|
1084static void
1085lapic_set_svr(struct vlapic *vlapic, uint32_t new)
| 1071void
1072vlapic_svr_write_handler(struct vlapic *vlapic)
|
1086{
1087 struct LAPIC *lapic;
| 1073{
1074 struct LAPIC *lapic;
|
1088 uint32_t old, changed;
| 1075 uint32_t old, new, changed;
|
1089
| 1076
|
1090 lapic = &vlapic->apic;
1091 old = lapic->svr;
| 1077 lapic = vlapic->apic_page;
1078
1079 new = lapic->svr;
1080 old = vlapic->svr_last;
1081 vlapic->svr_last = new;
1082
|
1092 changed = old ^ new;
1093 if ((changed & APIC_SVR_ENABLE) != 0) {
1094 if ((new & APIC_SVR_ENABLE) == 0) {
1095 /*
| 1083 changed = old ^ new;
1084 if ((changed & APIC_SVR_ENABLE) != 0) {
1085 if ((new & APIC_SVR_ENABLE) == 0) {
1086 /*
|
1096 * The apic is now disabled so stop the apic timer.
| 1087 * The apic is now disabled so stop the apic timer
1088 * and mask all the LVT entries.
|
1097 */
1098 VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1099 VLAPIC_TIMER_LOCK(vlapic);
1100 callout_stop(&vlapic->callout);
1101 VLAPIC_TIMER_UNLOCK(vlapic);
| 1089 */
1090 VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1091 VLAPIC_TIMER_LOCK(vlapic);
1092 callout_stop(&vlapic->callout);
1093 VLAPIC_TIMER_UNLOCK(vlapic);
|
| 1094 vlapic_mask_lvts(vlapic);
|
1102 } else {
1103 /*
1104 * The apic is now enabled so restart the apic timer
1105 * if it is configured in periodic mode.
1106 */
1107 VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1108 if (vlapic_periodic_timer(vlapic))
| 1095 } else {
1096 /*
1097 * The apic is now enabled so restart the apic timer
1098 * if it is configured in periodic mode.
1099 */
1100 VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1101 if (vlapic_periodic_timer(vlapic))
|
1109 vlapic_set_icr_timer(vlapic, lapic->icr_timer);
| 1102 vlapic_icrtmr_write_handler(vlapic);
|
1110 }
1111 }
| 1103 }
1104 }
|
1112 lapic->svr = new;
| |
1113}
1114
1115int
1116vlapic_read(struct vlapic *vlapic, uint64_t offset, uint64_t *data, bool *retu)
1117{
| 1105}
1106
1107int
1108vlapic_read(struct vlapic *vlapic, uint64_t offset, uint64_t *data, bool *retu)
1109{
|
1118 struct LAPIC *lapic = &vlapic->apic;
| 1110 struct LAPIC *lapic = vlapic->apic_page;
|
1119 uint32_t *reg;
1120 int i;
1121
1122 if (offset > sizeof(*lapic)) {
1123 *data = 0;
1124 goto done;
1125 }
1126
1127 offset &= ~3;
1128 switch(offset)
1129 {
1130 case APIC_OFFSET_ID:
| 1111 uint32_t *reg;
1112 int i;
1113
1114 if (offset > sizeof(*lapic)) {
1115 *data = 0;
1116 goto done;
1117 }
1118
1119 offset &= ~3;
1120 switch(offset)
1121 {
1122 case APIC_OFFSET_ID:
|
1131 *data = vlapic_get_id(vlapic);
| 1123 *data = lapic->id;
|
1132 break;
1133 case APIC_OFFSET_VER:
1134 *data = lapic->version;
1135 break;
1136 case APIC_OFFSET_TPR:
1137 *data = lapic->tpr;
1138 break;
1139 case APIC_OFFSET_APR:
1140 *data = lapic->apr;
1141 break;
1142 case APIC_OFFSET_PPR:
1143 *data = lapic->ppr;
1144 break;
1145 case APIC_OFFSET_EOI:
1146 *data = lapic->eoi;
1147 break;
1148 case APIC_OFFSET_LDR:
| 1124 break;
1125 case APIC_OFFSET_VER:
1126 *data = lapic->version;
1127 break;
1128 case APIC_OFFSET_TPR:
1129 *data = lapic->tpr;
1130 break;
1131 case APIC_OFFSET_APR:
1132 *data = lapic->apr;
1133 break;
1134 case APIC_OFFSET_PPR:
1135 *data = lapic->ppr;
1136 break;
1137 case APIC_OFFSET_EOI:
1138 *data = lapic->eoi;
1139 break;
1140 case APIC_OFFSET_LDR:
|
1149 *data = vlapic_get_ldr(vlapic);
| 1141 *data = lapic->ldr;
|
1150 break;
1151 case APIC_OFFSET_DFR:
| 1142 break;
1143 case APIC_OFFSET_DFR:
|
1152 *data = vlapic_get_dfr(vlapic);
| 1144 *data = lapic->dfr;
|
1153 break;
1154 case APIC_OFFSET_SVR:
1155 *data = lapic->svr;
1156 break;
1157 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1158 i = (offset - APIC_OFFSET_ISR0) >> 2;
1159 reg = &lapic->isr0;
1160 *data = *(reg + i);
1161 break;
1162 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1163 i = (offset - APIC_OFFSET_TMR0) >> 2;
1164 reg = &lapic->tmr0;
1165 *data = *(reg + i);
1166 break;
1167 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1168 i = (offset - APIC_OFFSET_IRR0) >> 2;
1169 reg = &lapic->irr0;
1170 *data = atomic_load_acq_int(reg + i);
1171 break;
1172 case APIC_OFFSET_ESR:
1173 *data = lapic->esr;
1174 break;
1175 case APIC_OFFSET_ICR_LOW:
1176 *data = lapic->icr_lo;
| 1145 break;
1146 case APIC_OFFSET_SVR:
1147 *data = lapic->svr;
1148 break;
1149 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1150 i = (offset - APIC_OFFSET_ISR0) >> 2;
1151 reg = &lapic->isr0;
1152 *data = *(reg + i);
1153 break;
1154 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1155 i = (offset - APIC_OFFSET_TMR0) >> 2;
1156 reg = &lapic->tmr0;
1157 *data = *(reg + i);
1158 break;
1159 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1160 i = (offset - APIC_OFFSET_IRR0) >> 2;
1161 reg = &lapic->irr0;
1162 *data = atomic_load_acq_int(reg + i);
1163 break;
1164 case APIC_OFFSET_ESR:
1165 *data = lapic->esr;
1166 break;
1167 case APIC_OFFSET_ICR_LOW:
1168 *data = lapic->icr_lo;
|
| 1169 if (x2apic(vlapic))
1170 *data |= (uint64_t)lapic->icr_hi << 32;
|
1177 break;
1178 case APIC_OFFSET_ICR_HI:
1179 *data = lapic->icr_hi;
1180 break;
1181 case APIC_OFFSET_CMCI_LVT:
1182 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1183 *data = vlapic_get_lvt(vlapic, offset);
| 1171 break;
1172 case APIC_OFFSET_ICR_HI:
1173 *data = lapic->icr_hi;
1174 break;
1175 case APIC_OFFSET_CMCI_LVT:
1176 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1177 *data = vlapic_get_lvt(vlapic, offset);
|
| 1178#ifdef INVARIANTS
1179 reg = vlapic_get_lvtptr(vlapic, offset);
1180 KASSERT(*data == *reg, ("inconsistent lvt value at "
1181 "offset %#lx: %#lx/%#x", offset, *data, *reg));
1182#endif
|
1184 break;
| 1183 break;
|
1185 case APIC_OFFSET_ICR:
| 1184 case APIC_OFFSET_TIMER_ICR:
|
1186 *data = lapic->icr_timer;
1187 break;
| 1185 *data = lapic->icr_timer;
1186 break;
|
1188 case APIC_OFFSET_CCR:
| 1187 case APIC_OFFSET_TIMER_CCR:
|
1189 *data = vlapic_get_ccr(vlapic);
1190 break;
| 1188 *data = vlapic_get_ccr(vlapic);
1189 break;
|
1191 case APIC_OFFSET_DCR:
| 1190 case APIC_OFFSET_TIMER_DCR:
|
1192 *data = lapic->dcr_timer;
1193 break;
1194 case APIC_OFFSET_RRR:
1195 default:
1196 *data = 0;
1197 break;
1198 }
1199done:
1200 VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1201 return 0;
1202}
1203
1204int
1205vlapic_write(struct vlapic *vlapic, uint64_t offset, uint64_t data, bool *retu)
1206{
| 1191 *data = lapic->dcr_timer;
1192 break;
1193 case APIC_OFFSET_RRR:
1194 default:
1195 *data = 0;
1196 break;
1197 }
1198done:
1199 VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1200 return 0;
1201}
1202
1203int
1204vlapic_write(struct vlapic *vlapic, uint64_t offset, uint64_t data, bool *retu)
1205{
|
1207 struct LAPIC *lapic = &vlapic->apic;
| 1206 struct LAPIC *lapic = vlapic->apic_page;
1207 uint32_t *regptr;
|
1208 int retval;
1209
| 1208 int retval;
1209
|
| 1210 KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
1211 ("vlapic_write: invalid offset %#lx", offset));
1212
|
1210 VLAPIC_CTR2(vlapic, "vlapic write offset %#x, data %#lx", offset, data);
1211
1212 if (offset > sizeof(*lapic)) {
1213 return 0;
1214 }
1215
1216 retval = 0;
| 1213 VLAPIC_CTR2(vlapic, "vlapic write offset %#x, data %#lx", offset, data);
1214
1215 if (offset > sizeof(*lapic)) {
1216 return 0;
1217 }
1218
1219 retval = 0;
|
1217 offset &= ~3;
| |
1218 switch(offset)
1219 {
1220 case APIC_OFFSET_ID:
| 1220 switch(offset)
1221 {
1222 case APIC_OFFSET_ID:
|
| 1223 lapic->id = data;
1224 vlapic_id_write_handler(vlapic);
|
1221 break;
1222 case APIC_OFFSET_TPR:
1223 lapic->tpr = data & 0xff;
1224 vlapic_update_ppr(vlapic);
1225 break;
1226 case APIC_OFFSET_EOI:
1227 vlapic_process_eoi(vlapic);
1228 break;
1229 case APIC_OFFSET_LDR:
| 1225 break;
1226 case APIC_OFFSET_TPR:
1227 lapic->tpr = data & 0xff;
1228 vlapic_update_ppr(vlapic);
1229 break;
1230 case APIC_OFFSET_EOI:
1231 vlapic_process_eoi(vlapic);
1232 break;
1233 case APIC_OFFSET_LDR:
|
1230 vlapic_set_ldr(vlapic, data);
| 1234 lapic->ldr = data;
1235 vlapic_ldr_write_handler(vlapic);
|
1231 break;
1232 case APIC_OFFSET_DFR:
| 1236 break;
1237 case APIC_OFFSET_DFR:
|
1233 vlapic_set_dfr(vlapic, data);
| 1238 lapic->dfr = data;
1239 vlapic_dfr_write_handler(vlapic);
|
1234 break;
1235 case APIC_OFFSET_SVR:
| 1240 break;
1241 case APIC_OFFSET_SVR:
|
1236 lapic_set_svr(vlapic, data);
| 1242 lapic->svr = data;
1243 vlapic_svr_write_handler(vlapic);
|
1237 break;
1238 case APIC_OFFSET_ICR_LOW:
| 1244 break;
1245 case APIC_OFFSET_ICR_LOW:
|
1239 if (!x2apic(vlapic)) {
1240 data &= 0xffffffff;
1241 data |= (uint64_t)lapic->icr_hi << 32;
1242 }
1243 retval = lapic_process_icr(vlapic, data, retu);
| 1246 lapic->icr_lo = data;
1247 if (x2apic(vlapic))
1248 lapic->icr_hi = data >> 32;
1249 retval = vlapic_icrlo_write_handler(vlapic, retu);
|
1244 break;
1245 case APIC_OFFSET_ICR_HI:
| 1250 break;
1251 case APIC_OFFSET_ICR_HI:
|
1246 if (!x2apic(vlapic)) {
1247 retval = 0;
1248 lapic->icr_hi = data;
1249 }
| 1252 lapic->icr_hi = data;
|
1250 break;
1251 case APIC_OFFSET_CMCI_LVT:
1252 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
| 1253 break;
1254 case APIC_OFFSET_CMCI_LVT:
1255 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
|
1253 vlapic_set_lvt(vlapic, offset, data);
| 1256 regptr = vlapic_get_lvtptr(vlapic, offset);
1257 *regptr = data;
1258 vlapic_lvt_write_handler(vlapic, offset);
|
1254 break;
| 1259 break;
|
1255 case APIC_OFFSET_ICR:
1256 vlapic_set_icr_timer(vlapic, data);
| 1260 case APIC_OFFSET_TIMER_ICR:
1261 lapic->icr_timer = data;
1262 vlapic_icrtmr_write_handler(vlapic);
|
1257 break;
1258
| 1263 break;
1264
|
1259 case APIC_OFFSET_DCR:
1260 vlapic_set_dcr(vlapic, data);
| 1265 case APIC_OFFSET_TIMER_DCR:
1266 lapic->dcr_timer = data;
1267 vlapic_dcr_write_handler(vlapic);
|
1261 break;
1262
1263 case APIC_OFFSET_ESR:
| 1268 break;
1269
1270 case APIC_OFFSET_ESR:
|
1264 vlapic_update_errors(vlapic);
| 1271 vlapic_esr_write_handler(vlapic);
|
1265 break;
1266 case APIC_OFFSET_VER:
1267 case APIC_OFFSET_APR:
1268 case APIC_OFFSET_PPR:
1269 case APIC_OFFSET_RRR:
1270 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1271 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1272 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
| 1272 break;
1273 case APIC_OFFSET_VER:
1274 case APIC_OFFSET_APR:
1275 case APIC_OFFSET_PPR:
1276 case APIC_OFFSET_RRR:
1277 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1278 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1279 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
|
1273 case APIC_OFFSET_CCR:
| 1280 case APIC_OFFSET_TIMER_CCR:
|
1274 default:
1275 // Read only.
1276 break;
1277 }
1278
1279 return (retval);
1280}
1281
| 1281 default:
1282 // Read only.
1283 break;
1284 }
1285
1286 return (retval);
1287}
1288
|
1282struct vlapic *
1283vlapic_init(struct vm *vm, int vcpuid)
| 1289static void
1290vlapic_reset(struct vlapic *vlapic)
|
1284{
| 1291{
|
1285 struct vlapic *vlapic;
| 1292 struct LAPIC *lapic;
1293
1294 lapic = vlapic->apic_page;
1295 bzero(lapic, sizeof(struct LAPIC));
|
1286
| 1296
|
1287 vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO);
1288 vlapic->vm = vm;
1289 vlapic->vcpuid = vcpuid;
| 1297 lapic->id = vlapic_get_id(vlapic);
1298 lapic->version = VLAPIC_VERSION;
1299 lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
1300 lapic->dfr = 0xffffffff;
1301 lapic->svr = APIC_SVR_VECTOR;
1302 vlapic_mask_lvts(vlapic);
1303 vlapic_reset_tmr(vlapic);
|
1290
| 1304
|
| 1305 lapic->dcr_timer = 0;
1306 vlapic_dcr_write_handler(vlapic);
1307
1308 if (vlapic->vcpuid == 0)
1309 vlapic->boot_state = BS_RUNNING; /* BSP */
1310 else
1311 vlapic->boot_state = BS_INIT; /* AP */
1312
1313 vlapic->svr_last = lapic->svr;
1314}
1315
1316void
1317vlapic_init(struct vlapic *vlapic)
1318{
1319 KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
1320 KASSERT(vlapic->vcpuid >= 0 && vlapic->vcpuid < VM_MAXCPU,
1321 ("vlapic_init: vcpuid is not initialized"));
1322 KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
1323 "initialized"));
1324
|
1291 /*
1292 * If the vlapic is configured in x2apic mode then it will be
1293 * accessed in the critical section via the MSR emulation code.
1294 *
1295 * Therefore the timer mutex must be a spinlock because blockable
1296 * mutexes cannot be acquired in a critical section.
1297 */
1298 mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1299 callout_init(&vlapic->callout, 1);
1300
1301 vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1302
| 1325 /*
1326 * If the vlapic is configured in x2apic mode then it will be
1327 * accessed in the critical section via the MSR emulation code.
1328 *
1329 * Therefore the timer mutex must be a spinlock because blockable
1330 * mutexes cannot be acquired in a critical section.
1331 */
1332 mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1333 callout_init(&vlapic->callout, 1);
1334
1335 vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1336
|
1303 if (vcpuid == 0)
| 1337 if (vlapic->vcpuid == 0)
|
1304 vlapic->msr_apicbase |= APICBASE_BSP;
1305
1306 vlapic_reset(vlapic);
| 1338 vlapic->msr_apicbase |= APICBASE_BSP;
1339
1340 vlapic_reset(vlapic);
|
1307
1308 return (vlapic);
| |
1309}
1310
1311void
1312vlapic_cleanup(struct vlapic *vlapic)
1313{
1314
1315 callout_drain(&vlapic->callout);
| 1341}
1342
1343void
1344vlapic_cleanup(struct vlapic *vlapic)
1345{
1346
1347 callout_drain(&vlapic->callout);
|
1316 free(vlapic, M_VLAPIC);
| |
1317}
1318
1319uint64_t
1320vlapic_get_apicbase(struct vlapic *vlapic)
1321{
1322
1323 return (vlapic->msr_apicbase);
1324}
1325
1326void
| 1348}
1349
1350uint64_t
1351vlapic_get_apicbase(struct vlapic *vlapic)
1352{
1353
1354 return (vlapic->msr_apicbase);
1355}
1356
1357void
|
1327vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val)
| 1358vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
|
1328{
| 1359{
|
1329 int err;
| 1360 struct LAPIC *lapic;
|
1330 enum x2apic_state state;
| 1361 enum x2apic_state state;
|
| 1362 uint64_t old;
1363 int err;
|
1331
1332 err = vm_get_x2apic_state(vlapic->vm, vlapic->vcpuid, &state);
1333 if (err)
1334 panic("vlapic_set_apicbase: err %d fetching x2apic state", err);
1335
1336 if (state == X2APIC_DISABLED)
| 1364
1365 err = vm_get_x2apic_state(vlapic->vm, vlapic->vcpuid, &state);
1366 if (err)
1367 panic("vlapic_set_apicbase: err %d fetching x2apic state", err);
1368
1369 if (state == X2APIC_DISABLED)
|
1337 val &= ~APICBASE_X2APIC;
| 1370 new &= ~APICBASE_X2APIC;
|
1338
| 1371
|
1339 vlapic->msr_apicbase = val;
| 1372 old = vlapic->msr_apicbase;
1373 vlapic->msr_apicbase = new;
1374
1375 /*
1376 * If the vlapic is switching between xAPIC and x2APIC modes then
1377 * reset the mode-dependent registers.
1378 */
1379 if ((old ^ new) & APICBASE_X2APIC) {
1380 lapic = vlapic->apic_page;
1381 lapic->id = vlapic_get_id(vlapic);
1382 if (x2apic(vlapic)) {
1383 lapic->ldr = x2apic_ldr(vlapic);
1384 lapic->dfr = 0;
1385 } else {
1386 lapic->ldr = 0;
1387 lapic->dfr = 0xffffffff;
1388 }
1389 }
|
1340}
1341
1342void
1343vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1344{
1345 struct vlapic *vlapic;
1346
1347 vlapic = vm_lapic(vm, vcpuid);
1348
1349 if (state == X2APIC_DISABLED)
1350 vlapic->msr_apicbase &= ~APICBASE_X2APIC;
1351}
1352
1353void
1354vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
1355 int delmode, int vec)
1356{
1357 bool lowprio;
1358 int vcpuid;
1359 cpuset_t dmask;
1360
1361 if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
1362 VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
1363 return;
1364 }
1365 lowprio = (delmode == APIC_DELMODE_LOWPRIO);
1366
1367 /*
1368 * We don't provide any virtual interrupt redirection hardware so
1369 * all interrupts originating from the ioapic or MSI specify the
1370 * 'dest' in the legacy xAPIC format.
1371 */
1372 vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
1373
1374 while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1375 vcpuid--;
1376 CPU_CLR(vcpuid, &dmask);
1377 lapic_set_intr(vm, vcpuid, vec, level);
1378 }
1379}
1380
| 1390}
1391
1392void
1393vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1394{
1395 struct vlapic *vlapic;
1396
1397 vlapic = vm_lapic(vm, vcpuid);
1398
1399 if (state == X2APIC_DISABLED)
1400 vlapic->msr_apicbase &= ~APICBASE_X2APIC;
1401}
1402
1403void
1404vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
1405 int delmode, int vec)
1406{
1407 bool lowprio;
1408 int vcpuid;
1409 cpuset_t dmask;
1410
1411 if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
1412 VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
1413 return;
1414 }
1415 lowprio = (delmode == APIC_DELMODE_LOWPRIO);
1416
1417 /*
1418 * We don't provide any virtual interrupt redirection hardware so
1419 * all interrupts originating from the ioapic or MSI specify the
1420 * 'dest' in the legacy xAPIC format.
1421 */
1422 vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
1423
1424 while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1425 vcpuid--;
1426 CPU_CLR(vcpuid, &dmask);
1427 lapic_set_intr(vm, vcpuid, vec, level);
1428 }
1429}
1430
|
| 1431void
1432vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
1433{
1434 /*
1435 * Post an interrupt to the vcpu currently running on 'hostcpu'.
1436 *
1437 * This is done by leveraging features like Posted Interrupts (Intel)
1438 * Doorbell MSR (AMD AVIC) that avoid a VM exit.
1439 *
1440 * If neither of these features are available then fallback to
1441 * sending an IPI to 'hostcpu'.
1442 */
1443 if (vlapic->ops.post_intr)
1444 (*vlapic->ops.post_intr)(vlapic, hostcpu);
1445 else
1446 ipi_cpu(hostcpu, ipinum);
1447}
1448
|
1381bool
1382vlapic_enabled(struct vlapic *vlapic)
1383{
| 1449bool
1450vlapic_enabled(struct vlapic *vlapic)
1451{
|
1384 struct LAPIC *lapic = &vlapic->apic;
| 1452 struct LAPIC *lapic = vlapic->apic_page;
|
1385
1386 if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1387 (lapic->svr & APIC_SVR_ENABLE) != 0)
1388 return (true);
1389 else
1390 return (false);
1391}
| 1453
1454 if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1455 (lapic->svr & APIC_SVR_ENABLE) != 0)
1456 return (true);
1457 else
1458 return (false);
1459}
|
| 1460
1461static void
1462vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
1463{
1464 struct LAPIC *lapic;
1465 uint32_t *tmrptr, mask;
1466 int idx;
1467
1468 lapic = vlapic->apic_page;
1469 tmrptr = &lapic->tmr0;
1470 idx = (vector / 32) * 4;
1471 mask = 1 << (vector % 32);
1472 if (level)
1473 tmrptr[idx] |= mask;
1474 else
1475 tmrptr[idx] &= ~mask;
1476
1477 if (vlapic->ops.set_tmr != NULL)
1478 (*vlapic->ops.set_tmr)(vlapic, vector, level);
1479}
1480
1481void
1482vlapic_reset_tmr(struct vlapic *vlapic)
1483{
1484 int vector;
1485
1486 VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
1487
1488 for (vector = 0; vector <= 255; vector++)
1489 vlapic_set_tmr(vlapic, vector, false);
1490}
1491
1492void
1493vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
1494 int delmode, int vector)
1495{
1496 cpuset_t dmask;
1497 bool lowprio;
1498
1499 KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
1500
1501 /*
1502 * A level trigger is valid only for fixed and lowprio delivery modes.
1503 */
1504 if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
1505 VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
1506 "delivery-mode %d", delmode);
1507 return;
1508 }
1509
1510 lowprio = (delmode == APIC_DELMODE_LOWPRIO);
1511 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
1512
1513 if (!CPU_ISSET(vlapic->vcpuid, &dmask))
1514 return;
1515
1516 VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
1517 vlapic_set_tmr(vlapic, vector, true);
1518}
|
| |