1/*-
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 268953 2014-07-21 19:08:02Z jhb $
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 268953 2014-07-21 19:08:02Z jhb $");
31
32#include <sys/param.h>
33#include <sys/lock.h>
34#include <sys/kernel.h>
35#include <sys/malloc.h>
36#include <sys/mutex.h>
37#include <sys/systm.h>
38#include <sys/smp.h>
39
40#include <x86/specialreg.h>
41#include <x86/apicreg.h>
42
43#include <machine/clock.h>
44#include <machine/smp.h>
45
46#include <machine/vmm.h>
47
48#include "vmm_ipi.h"
49#include "vmm_lapic.h"
50#include "vmm_ktr.h"
51#include "vmm_stat.h"
52
53#include "vlapic.h"
54#include "vlapic_priv.h"
55#include "vioapic.h"
56
57#define PRIO(x) ((x) >> 4)
58
59#define VLAPIC_VERSION (16)
60
61#define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
62
63/*
64 * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
65 * vlapic_callout_handler() and vcpu accesses to:
66 * - timer_freq_bt, timer_period_bt, timer_fire_bt
67 * - timer LVT register
68 */
69#define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx))
70#define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx))
71#define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx))
72
73/*
74 * APIC timer frequency:
75 * - arbitrary but chosen to be in the ballpark of contemporary hardware.
76 * - power-of-two to avoid loss of precision when converted to a bintime.
77 */
78#define VLAPIC_BUS_FREQ (128 * 1024 * 1024)
79
80static __inline uint32_t
81vlapic_get_id(struct vlapic *vlapic)
82{
83
84 if (x2apic(vlapic))
85 return (vlapic->vcpuid);
86 else
87 return (vlapic->vcpuid << 24);
88}
89
90static uint32_t
91x2apic_ldr(struct vlapic *vlapic)
92{
93 int apicid;
94 uint32_t ldr;
95
96 apicid = vlapic_get_id(vlapic);
97 ldr = 1 << (apicid & 0xf);
98 ldr |= (apicid & 0xffff0) << 12;
99 return (ldr);
100}
101
102void
103vlapic_dfr_write_handler(struct vlapic *vlapic)
104{
105 struct LAPIC *lapic;
106
107 lapic = vlapic->apic_page;
108 if (x2apic(vlapic)) {
109 VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
110 lapic->dfr);
111 lapic->dfr = 0;
112 return;
113 }
114
115 lapic->dfr &= APIC_DFR_MODEL_MASK;
116 lapic->dfr |= APIC_DFR_RESERVED;
117
118 if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
119 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
120 else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
121 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
122 else
123 VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
124}
125
126void
127vlapic_ldr_write_handler(struct vlapic *vlapic)
128{
129 struct LAPIC *lapic;
130
131 lapic = vlapic->apic_page;
132
133 /* LDR is read-only in x2apic mode */
134 if (x2apic(vlapic)) {
135 VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
136 lapic->ldr);
137 lapic->ldr = x2apic_ldr(vlapic);
138 } else {
139 lapic->ldr &= ~APIC_LDR_RESERVED;
140 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
141 }
142}
143
144void
145vlapic_id_write_handler(struct vlapic *vlapic)
146{
147 struct LAPIC *lapic;
148
149 /*
150 * We don't allow the ID register to be modified so reset it back to
151 * its default value.
152 */
153 lapic = vlapic->apic_page;
154 lapic->id = vlapic_get_id(vlapic);
155}
156
157static int
158vlapic_timer_divisor(uint32_t dcr)
159{
160 switch (dcr & 0xB) {
161 case APIC_TDCR_1:
162 return (1);
163 case APIC_TDCR_2:
164 return (2);
165 case APIC_TDCR_4:
166 return (4);
167 case APIC_TDCR_8:
168 return (8);
169 case APIC_TDCR_16:
170 return (16);
171 case APIC_TDCR_32:
172 return (32);
173 case APIC_TDCR_64:
174 return (64);
175 case APIC_TDCR_128:
176 return (128);
177 default:
178 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
179 }
180}
181
182#if 0
183static inline void
184vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
185{
186 printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
187 *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
188 *lvt & APIC_LVTT_M);
189}
190#endif
191
192static uint32_t
193vlapic_get_ccr(struct vlapic *vlapic)
194{
195 struct bintime bt_now, bt_rem;
196 struct LAPIC *lapic;
197 uint32_t ccr;
198
199 ccr = 0;
200 lapic = vlapic->apic_page;
201
202 VLAPIC_TIMER_LOCK(vlapic);
203 if (callout_active(&vlapic->callout)) {
204 /*
205 * If the timer is scheduled to expire in the future then
206 * compute the value of 'ccr' based on the remaining time.
207 */
208 binuptime(&bt_now);
209 if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
210 bt_rem = vlapic->timer_fire_bt;
211 bintime_sub(&bt_rem, &bt_now);
212 ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
213 ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
214 }
215 }
216 KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
217 "icr_timer is %#x", ccr, lapic->icr_timer));
218 VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
219 ccr, lapic->icr_timer);
220 VLAPIC_TIMER_UNLOCK(vlapic);
221 return (ccr);
222}
223
224void
225vlapic_dcr_write_handler(struct vlapic *vlapic)
226{
227 struct LAPIC *lapic;
228 int divisor;
229
230 lapic = vlapic->apic_page;
231 VLAPIC_TIMER_LOCK(vlapic);
232
233 divisor = vlapic_timer_divisor(lapic->dcr_timer);
234 VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
235 lapic->dcr_timer, divisor);
236
237 /*
238 * Update the timer frequency and the timer period.
239 *
240 * XXX changes to the frequency divider will not take effect until
241 * the timer is reloaded.
242 */
243 FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
244 vlapic->timer_period_bt = vlapic->timer_freq_bt;
245 bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
246
247 VLAPIC_TIMER_UNLOCK(vlapic);
248}
249
250void
251vlapic_esr_write_handler(struct vlapic *vlapic)
252{
253 struct LAPIC *lapic;
254
255 lapic = vlapic->apic_page;
256 lapic->esr = vlapic->esr_pending;
257 vlapic->esr_pending = 0;
258}
259
260int
261vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
262{
263 struct LAPIC *lapic;
264 uint32_t *irrptr, *tmrptr, mask;
265 int idx;
266
267 KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
268
269 lapic = vlapic->apic_page;
270 if (!(lapic->svr & APIC_SVR_ENABLE)) {
271 VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
272 "interrupt %d", vector);
273 return (0);
274 }
275
276 if (vector < 16) {
277 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
278 VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
279 vector);
280 return (1);
281 }
282
283 if (vlapic->ops.set_intr_ready)
284 return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
285
286 idx = (vector / 32) * 4;
287 mask = 1 << (vector % 32);
288
289 irrptr = &lapic->irr0;
290 atomic_set_int(&irrptr[idx], mask);
291
292 /*
293 * Verify that the trigger-mode of the interrupt matches with
294 * the vlapic TMR registers.
295 */
296 tmrptr = &lapic->tmr0;
297 if ((tmrptr[idx] & mask) != (level ? mask : 0)) {
298 VLAPIC_CTR3(vlapic, "vlapic TMR[%d] is 0x%08x but "
299 "interrupt is %s-triggered", idx / 4, tmrptr[idx],
300 level ? "level" : "edge");
301 }
302
303 VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
304 return (1);
305}
306
307static __inline uint32_t *
308vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
309{
310 struct LAPIC *lapic = vlapic->apic_page;
311 int i;
312
313 switch (offset) {
314 case APIC_OFFSET_CMCI_LVT:
315 return (&lapic->lvt_cmci);
316 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
317 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
318 return ((&lapic->lvt_timer) + i);;
319 default:
320 panic("vlapic_get_lvt: invalid LVT\n");
321 }
322}
323
324static __inline int
325lvt_off_to_idx(uint32_t offset)
326{
327 int index;
328
329 switch (offset) {
330 case APIC_OFFSET_CMCI_LVT:
331 index = APIC_LVT_CMCI;
332 break;
333 case APIC_OFFSET_TIMER_LVT:
334 index = APIC_LVT_TIMER;
335 break;
336 case APIC_OFFSET_THERM_LVT:
337 index = APIC_LVT_THERMAL;
338 break;
339 case APIC_OFFSET_PERF_LVT:
340 index = APIC_LVT_PMC;
341 break;
342 case APIC_OFFSET_LINT0_LVT:
343 index = APIC_LVT_LINT0;
344 break;
345 case APIC_OFFSET_LINT1_LVT:
346 index = APIC_LVT_LINT1;
347 break;
348 case APIC_OFFSET_ERROR_LVT:
349 index = APIC_LVT_ERROR;
350 break;
351 default:
352 index = -1;
353 break;
354 }
355 KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
356 "invalid lvt index %d for offset %#x", index, offset));
357
358 return (index);
359}
360
361static __inline uint32_t
362vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
363{
364 int idx;
365 uint32_t val;
366
367 idx = lvt_off_to_idx(offset);
368 val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
369 return (val);
370}
371
372void
373vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
374{
375 uint32_t *lvtptr, mask, val;
376 struct LAPIC *lapic;
377 int idx;
378
379 lapic = vlapic->apic_page;
380 lvtptr = vlapic_get_lvtptr(vlapic, offset);
381 val = *lvtptr;
382 idx = lvt_off_to_idx(offset);
383
384 if (!(lapic->svr & APIC_SVR_ENABLE))
385 val |= APIC_LVT_M;
386 mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
387 switch (offset) {
388 case APIC_OFFSET_TIMER_LVT:
389 mask |= APIC_LVTT_TM;
390 break;
391 case APIC_OFFSET_ERROR_LVT:
392 break;
393 case APIC_OFFSET_LINT0_LVT:
394 case APIC_OFFSET_LINT1_LVT:
395 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
396 /* FALLTHROUGH */
397 default:
398 mask |= APIC_LVT_DM;
399 break;
400 }
401 val &= mask;
402 *lvtptr = val;
403 atomic_store_rel_32(&vlapic->lvt_last[idx], val);
404}
405
406static void
407vlapic_mask_lvts(struct vlapic *vlapic)
408{
409 struct LAPIC *lapic = vlapic->apic_page;
410
411 lapic->lvt_cmci |= APIC_LVT_M;
412 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
413
414 lapic->lvt_timer |= APIC_LVT_M;
415 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
416
417 lapic->lvt_thermal |= APIC_LVT_M;
418 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
419
420 lapic->lvt_pcint |= APIC_LVT_M;
421 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
422
423 lapic->lvt_lint0 |= APIC_LVT_M;
424 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
425
426 lapic->lvt_lint1 |= APIC_LVT_M;
427 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
428
429 lapic->lvt_error |= APIC_LVT_M;
430 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
431}
432
433static int
434vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
435{
436 uint32_t vec, mode;
437
438 if (lvt & APIC_LVT_M)
439 return (0);
440
441 vec = lvt & APIC_LVT_VECTOR;
442 mode = lvt & APIC_LVT_DM;
443
444 switch (mode) {
445 case APIC_LVT_DM_FIXED:
446 if (vec < 16) {
447 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
448 return (0);
449 }
450 if (vlapic_set_intr_ready(vlapic, vec, false))
451 vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
452 break;
453 case APIC_LVT_DM_NMI:
454 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
455 break;
456 case APIC_LVT_DM_EXTINT:
457 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
458 break;
459 default:
460 // Other modes ignored
461 return (0);
462 }
463 return (1);
464}
465
466#if 1
467static void
468dump_isrvec_stk(struct vlapic *vlapic)
469{
470 int i;
471 uint32_t *isrptr;
472
473 isrptr = &vlapic->apic_page->isr0;
474 for (i = 0; i < 8; i++)
475 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
476
477 for (i = 0; i <= vlapic->isrvec_stk_top; i++)
478 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
479}
480#endif
481
482/*
483 * Algorithm adopted from section "Interrupt, Task and Processor Priority"
484 * in Intel Architecture Manual Vol 3a.
485 */
486static void
487vlapic_update_ppr(struct vlapic *vlapic)
488{
489 int isrvec, tpr, ppr;
490
491 /*
492 * Note that the value on the stack at index 0 is always 0.
493 *
494 * This is a placeholder for the value of ISRV when none of the
495 * bits is set in the ISRx registers.
496 */
497 isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
498 tpr = vlapic->apic_page->tpr;
499
500#if 1
501 {
502 int i, lastprio, curprio, vector, idx;
503 uint32_t *isrptr;
504
505 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
506 panic("isrvec_stk is corrupted: %d", isrvec);
507
508 /*
509 * Make sure that the priority of the nested interrupts is
510 * always increasing.
511 */
512 lastprio = -1;
513 for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
514 curprio = PRIO(vlapic->isrvec_stk[i]);
515 if (curprio <= lastprio) {
516 dump_isrvec_stk(vlapic);
517 panic("isrvec_stk does not satisfy invariant");
518 }
519 lastprio = curprio;
520 }
521
522 /*
523 * Make sure that each bit set in the ISRx registers has a
524 * corresponding entry on the isrvec stack.
525 */
526 i = 1;
527 isrptr = &vlapic->apic_page->isr0;
528 for (vector = 0; vector < 256; vector++) {
529 idx = (vector / 32) * 4;
530 if (isrptr[idx] & (1 << (vector % 32))) {
531 if (i > vlapic->isrvec_stk_top ||
532 vlapic->isrvec_stk[i] != vector) {
533 dump_isrvec_stk(vlapic);
534 panic("ISR and isrvec_stk out of sync");
535 }
536 i++;
537 }
538 }
539 }
540#endif
541
542 if (PRIO(tpr) >= PRIO(isrvec))
543 ppr = tpr;
544 else
545 ppr = isrvec & 0xf0;
546
547 vlapic->apic_page->ppr = ppr;
548 VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
549}
550
551static void
552vlapic_process_eoi(struct vlapic *vlapic)
553{
554 struct LAPIC *lapic = vlapic->apic_page;
555 uint32_t *isrptr, *tmrptr;
556 int i, idx, bitpos, vector;
557
558 isrptr = &lapic->isr0;
559 tmrptr = &lapic->tmr0;
560
561 /*
562 * The x86 architecture reserves the the first 32 vectors for use
563 * by the processor.
564 */
565 for (i = 7; i > 0; i--) {
566 idx = i * 4;
567 bitpos = fls(isrptr[idx]);
568 if (bitpos-- != 0) {
569 if (vlapic->isrvec_stk_top <= 0) {
570 panic("invalid vlapic isrvec_stk_top %d",
571 vlapic->isrvec_stk_top);
572 }
573 isrptr[idx] &= ~(1 << bitpos);
574 VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
575 vlapic->isrvec_stk_top--;
576 vlapic_update_ppr(vlapic);
577 if ((tmrptr[idx] & (1 << bitpos)) != 0) {
578 vector = i * 32 + bitpos;
579 vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
580 vector);
581 }
582 return;
583 }
584 }
585}
586
587static __inline int
588vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
589{
590
591 return (lvt & mask);
592}
593
594static __inline int
595vlapic_periodic_timer(struct vlapic *vlapic)
596{
597 uint32_t lvt;
598
599 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
600
601 return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
602}
603
604static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
605
606void
607vlapic_set_error(struct vlapic *vlapic, uint32_t mask)
608{
609 uint32_t lvt;
610
611 vlapic->esr_pending |= mask;
612 if (vlapic->esr_firing)
613 return;
614 vlapic->esr_firing = 1;
615
616 // The error LVT always uses the fixed delivery mode.
617 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
618 if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
619 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1);
620 }
621 vlapic->esr_firing = 0;
622}
623
624static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
625
626static void
627vlapic_fire_timer(struct vlapic *vlapic)
628{
629 uint32_t lvt;
630
631 KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
632
633 // The timer LVT always uses the fixed delivery mode.
634 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
635 if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
636 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
637 }
638}
639
640static VMM_STAT(VLAPIC_INTR_CMC,
641 "corrected machine check interrupts generated by vlapic");
642
643void
644vlapic_fire_cmci(struct vlapic *vlapic)
645{
646 uint32_t lvt;
647
648 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
649 if (vlapic_fire_lvt(vlapic, lvt)) {
650 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
651 }
652}
653
654static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
655 "lvts triggered");
656
657int
658vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
659{
660 uint32_t lvt;
661
662 if (vlapic_enabled(vlapic) == false) {
663 /*
664 * When the local APIC is global/hardware disabled,
665 * LINT[1:0] pins are configured as INTR and NMI pins,
666 * respectively.
667 */
668 switch (vector) {
669 case APIC_LVT_LINT0:
670 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
671 break;
672 case APIC_LVT_LINT1:
673 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
674 break;
675 default:
676 break;
677 }
678 return (0);
679 }
680
681 switch (vector) {
682 case APIC_LVT_LINT0:
683 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT);
684 break;
685 case APIC_LVT_LINT1:
686 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT);
687 break;
688 case APIC_LVT_TIMER:
689 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
690 lvt |= APIC_LVT_DM_FIXED;
691 break;
692 case APIC_LVT_ERROR:
693 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
694 lvt |= APIC_LVT_DM_FIXED;
695 break;
696 case APIC_LVT_PMC:
697 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT);
698 break;
699 case APIC_LVT_THERMAL:
700 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT);
701 break;
702 case APIC_LVT_CMCI:
703 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
704 break;
705 default:
706 return (EINVAL);
707 }
708 if (vlapic_fire_lvt(vlapic, lvt)) {
709 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
710 LVTS_TRIGGERRED, vector, 1);
711 }
712 return (0);
713}
714
715static void
716vlapic_callout_handler(void *arg)
717{
718 struct vlapic *vlapic;
719 struct bintime bt, btnow;
720 sbintime_t rem_sbt;
721
722 vlapic = arg;
723
724 VLAPIC_TIMER_LOCK(vlapic);
725 if (callout_pending(&vlapic->callout)) /* callout was reset */
726 goto done;
727
728 if (!callout_active(&vlapic->callout)) /* callout was stopped */
729 goto done;
730
731 callout_deactivate(&vlapic->callout);
732
733 vlapic_fire_timer(vlapic);
734
735 if (vlapic_periodic_timer(vlapic)) {
736 binuptime(&btnow);
737 KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
738 ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
739 btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
740 vlapic->timer_fire_bt.frac));
741
742 /*
743 * Compute the delta between when the timer was supposed to
744 * fire and the present time.
745 */
746 bt = btnow;
747 bintime_sub(&bt, &vlapic->timer_fire_bt);
748
749 rem_sbt = bttosbt(vlapic->timer_period_bt);
750 if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
751 /*
752 * Adjust the time until the next countdown downward
753 * to account for the lost time.
754 */
755 rem_sbt -= bttosbt(bt);
756 } else {
757 /*
758 * If the delta is greater than the timer period then
759 * just reset our time base instead of trying to catch
760 * up.
761 */
762 vlapic->timer_fire_bt = btnow;
763 VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
764 "usecs, period is %lu usecs - resetting time base",
765 bttosbt(bt) / SBT_1US,
766 bttosbt(vlapic->timer_period_bt) / SBT_1US);
767 }
768
769 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
770 callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
771 vlapic_callout_handler, vlapic, 0);
772 }
773done:
774 VLAPIC_TIMER_UNLOCK(vlapic);
775}
776
777void
778vlapic_icrtmr_write_handler(struct vlapic *vlapic)
779{
780 struct LAPIC *lapic;
781 sbintime_t sbt;
782 uint32_t icr_timer;
783
784 VLAPIC_TIMER_LOCK(vlapic);
785
786 lapic = vlapic->apic_page;
787 icr_timer = lapic->icr_timer;
788
789 vlapic->timer_period_bt = vlapic->timer_freq_bt;
790 bintime_mul(&vlapic->timer_period_bt, icr_timer);
791
792 if (icr_timer != 0) {
793 binuptime(&vlapic->timer_fire_bt);
794 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
795
796 sbt = bttosbt(vlapic->timer_period_bt);
797 callout_reset_sbt(&vlapic->callout, sbt, 0,
798 vlapic_callout_handler, vlapic, 0);
799 } else
800 callout_stop(&vlapic->callout);
801
802 VLAPIC_TIMER_UNLOCK(vlapic);
803}
804
805/*
806 * This function populates 'dmask' with the set of vcpus that match the
807 * addressing specified by the (dest, phys, lowprio) tuple.
808 *
809 * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
810 * or xAPIC (8-bit) destination field.
811 */
812static void
813vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
814 bool lowprio, bool x2apic_dest)
815{
816 struct vlapic *vlapic;
817 uint32_t dfr, ldr, ldest, cluster;
818 uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
819 cpuset_t amask;
820 int vcpuid;
821
822 if ((x2apic_dest && dest == 0xffffffff) ||
823 (!x2apic_dest && dest == 0xff)) {
824 /*
825 * Broadcast in both logical and physical modes.
826 */
827 *dmask = vm_active_cpus(vm);
828 return;
829 }
830
831 if (phys) {
832 /*
833 * Physical mode: destination is APIC ID.
834 */
835 CPU_ZERO(dmask);
836 vcpuid = vm_apicid2vcpuid(vm, dest);
837 if (vcpuid < VM_MAXCPU)
838 CPU_SET(vcpuid, dmask);
839 } else {
840 /*
841 * In the "Flat Model" the MDA is interpreted as an 8-bit wide
842 * bitmask. This model is only avilable in the xAPIC mode.
843 */
844 mda_flat_ldest = dest & 0xff;
845
846 /*
847 * In the "Cluster Model" the MDA is used to identify a
848 * specific cluster and a set of APICs in that cluster.
849 */
850 if (x2apic_dest) {
851 mda_cluster_id = dest >> 16;
852 mda_cluster_ldest = dest & 0xffff;
853 } else {
854 mda_cluster_id = (dest >> 4) & 0xf;
855 mda_cluster_ldest = dest & 0xf;
856 }
857
858 /*
859 * Logical mode: match each APIC that has a bit set
860 * in it's LDR that matches a bit in the ldest.
861 */
862 CPU_ZERO(dmask);
863 amask = vm_active_cpus(vm);
864 while ((vcpuid = CPU_FFS(&amask)) != 0) {
865 vcpuid--;
866 CPU_CLR(vcpuid, &amask);
867
868 vlapic = vm_lapic(vm, vcpuid);
869 dfr = vlapic->apic_page->dfr;
870 ldr = vlapic->apic_page->ldr;
871
872 if ((dfr & APIC_DFR_MODEL_MASK) ==
873 APIC_DFR_MODEL_FLAT) {
874 ldest = ldr >> 24;
875 mda_ldest = mda_flat_ldest;
876 } else if ((dfr & APIC_DFR_MODEL_MASK) ==
877 APIC_DFR_MODEL_CLUSTER) {
878 if (x2apic(vlapic)) {
879 cluster = ldr >> 16;
880 ldest = ldr & 0xffff;
881 } else {
882 cluster = ldr >> 28;
883 ldest = (ldr >> 24) & 0xf;
884 }
885 if (cluster != mda_cluster_id)
886 continue;
887 mda_ldest = mda_cluster_ldest;
888 } else {
889 /*
890 * Guest has configured a bad logical
891 * model for this vcpu - skip it.
892 */
893 VLAPIC_CTR1(vlapic, "vlapic has bad logical "
894 "model %x - cannot deliver interrupt", dfr);
895 continue;
896 }
897
898 if ((mda_ldest & ldest) != 0) {
899 CPU_SET(vcpuid, dmask);
900 if (lowprio)
901 break;
902 }
903 }
904 }
905}
906
907static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
908
909int
910vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
911{
912 int i;
913 bool phys;
914 cpuset_t dmask;
915 uint64_t icrval;
916 uint32_t dest, vec, mode;
917 struct vlapic *vlapic2;
918 struct vm_exit *vmexit;
919 struct LAPIC *lapic;
920
921 lapic = vlapic->apic_page;
922 lapic->icr_lo &= ~APIC_DELSTAT_PEND;
923 icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
924
925 if (x2apic(vlapic))
926 dest = icrval >> 32;
927 else
928 dest = icrval >> (32 + 24);
929 vec = icrval & APIC_VECTOR_MASK;
930 mode = icrval & APIC_DELMODE_MASK;
931
932 if (mode == APIC_DELMODE_FIXED && vec < 16) {
933 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
934 VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
935 return (0);
936 }
937
938 VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
939
940 if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
941 switch (icrval & APIC_DEST_MASK) {
942 case APIC_DEST_DESTFLD:
943 phys = ((icrval & APIC_DESTMODE_LOG) == 0);
944 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
945 x2apic(vlapic));
946 break;
947 case APIC_DEST_SELF:
948 CPU_SETOF(vlapic->vcpuid, &dmask);
949 break;
950 case APIC_DEST_ALLISELF:
951 dmask = vm_active_cpus(vlapic->vm);
952 break;
953 case APIC_DEST_ALLESELF:
954 dmask = vm_active_cpus(vlapic->vm);
955 CPU_CLR(vlapic->vcpuid, &dmask);
956 break;
957 default:
958 CPU_ZERO(&dmask); /* satisfy gcc */
959 break;
960 }
961
962 while ((i = CPU_FFS(&dmask)) != 0) {
963 i--;
964 CPU_CLR(i, &dmask);
965 if (mode == APIC_DELMODE_FIXED) {
966 lapic_intr_edge(vlapic->vm, i, vec);
967 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
968 IPIS_SENT, i, 1);
969 VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
970 "to vcpuid %d", vec, i);
971 } else {
972 vm_inject_nmi(vlapic->vm, i);
973 VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
974 "to vcpuid %d", i);
975 }
976 }
977
978 return (0); /* handled completely in the kernel */
979 }
980
981 if (mode == APIC_DELMODE_INIT) {
982 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
983 return (0);
984
985 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
986 vlapic2 = vm_lapic(vlapic->vm, dest);
987
988 /* move from INIT to waiting-for-SIPI state */
989 if (vlapic2->boot_state == BS_INIT) {
990 vlapic2->boot_state = BS_SIPI;
991 }
992
993 return (0);
994 }
995 }
996
997 if (mode == APIC_DELMODE_STARTUP) {
998 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
999 vlapic2 = vm_lapic(vlapic->vm, dest);
1000
1001 /*
1002 * Ignore SIPIs in any state other than wait-for-SIPI
1003 */
1004 if (vlapic2->boot_state != BS_SIPI)
1005 return (0);
1006
1007 /*
1008 * XXX this assumes that the startup IPI always succeeds
1009 */
1010 vlapic2->boot_state = BS_RUNNING;
1011 vm_activate_cpu(vlapic2->vm, dest);
1012
1013 *retu = true;
1014 vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
1015 vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
1016 vmexit->u.spinup_ap.vcpu = dest;
1017 vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
1018
1019 return (0);
1020 }
1021 }
1022
1023 /*
1024 * This will cause a return to userland.
1025 */
1026 return (1);
1027}
1028
1029void
1030vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val)
1031{
1032 int vec;
1033
1034 KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode"));
1035
1036 vec = val & 0xff;
1037 lapic_intr_edge(vlapic->vm, vlapic->vcpuid, vec);
1038 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, IPIS_SENT,
1039 vlapic->vcpuid, 1);
1040 VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec);
1041}
1042
1043int
1044vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
1045{
1046 struct LAPIC *lapic = vlapic->apic_page;
1047 int idx, i, bitpos, vector;
1048 uint32_t *irrptr, val;
1049
1050 if (vlapic->ops.pending_intr)
1051 return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
1052
1053 irrptr = &lapic->irr0;
1054
1055 /*
1056 * The x86 architecture reserves the the first 32 vectors for use
1057 * by the processor.
1058 */
1059 for (i = 7; i > 0; i--) {
1060 idx = i * 4;
1061 val = atomic_load_acq_int(&irrptr[idx]);
1062 bitpos = fls(val);
1063 if (bitpos != 0) {
1064 vector = i * 32 + (bitpos - 1);
1065 if (PRIO(vector) > PRIO(lapic->ppr)) {
1066 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
1067 if (vecptr != NULL)
1068 *vecptr = vector;
1069 return (1);
1070 } else
1071 break;
1072 }
1073 }
1074 return (0);
1075}
1076
1077void
1078vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1079{
1080 struct LAPIC *lapic = vlapic->apic_page;
1081 uint32_t *irrptr, *isrptr;
1082 int idx, stk_top;
1083
1084 if (vlapic->ops.intr_accepted)
1085 return ((*vlapic->ops.intr_accepted)(vlapic, vector));
1086
1087 /*
1088 * clear the ready bit for vector being accepted in irr
1089 * and set the vector as in service in isr.
1090 */
1091 idx = (vector / 32) * 4;
1092
1093 irrptr = &lapic->irr0;
1094 atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
1095 VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
1096
1097 isrptr = &lapic->isr0;
1098 isrptr[idx] |= 1 << (vector % 32);
1099 VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
1100
1101 /*
1102 * Update the PPR
1103 */
1104 vlapic->isrvec_stk_top++;
1105
1106 stk_top = vlapic->isrvec_stk_top;
1107 if (stk_top >= ISRVEC_STK_SIZE)
1108 panic("isrvec_stk_top overflow %d", stk_top);
1109
1110 vlapic->isrvec_stk[stk_top] = vector;
1111 vlapic_update_ppr(vlapic);
1112}
1113
1114void
1115vlapic_svr_write_handler(struct vlapic *vlapic)
1116{
1117 struct LAPIC *lapic;
1118 uint32_t old, new, changed;
1119
1120 lapic = vlapic->apic_page;
1121
1122 new = lapic->svr;
1123 old = vlapic->svr_last;
1124 vlapic->svr_last = new;
1125
1126 changed = old ^ new;
1127 if ((changed & APIC_SVR_ENABLE) != 0) {
1128 if ((new & APIC_SVR_ENABLE) == 0) {
1129 /*
1130 * The apic is now disabled so stop the apic timer
1131 * and mask all the LVT entries.
1132 */
1133 VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1134 VLAPIC_TIMER_LOCK(vlapic);
1135 callout_stop(&vlapic->callout);
1136 VLAPIC_TIMER_UNLOCK(vlapic);
1137 vlapic_mask_lvts(vlapic);
1138 } else {
1139 /*
1140 * The apic is now enabled so restart the apic timer
1141 * if it is configured in periodic mode.
1142 */
1143 VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1144 if (vlapic_periodic_timer(vlapic))
1145 vlapic_icrtmr_write_handler(vlapic);
1146 }
1147 }
1148}
1149
1150int
1151vlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1152 uint64_t *data, bool *retu)
1153{
1154 struct LAPIC *lapic = vlapic->apic_page;
1155 uint32_t *reg;
1156 int i;
1157
1158 /* Ignore MMIO accesses in x2APIC mode */
1159 if (x2apic(vlapic) && mmio_access) {
1160 VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode",
1161 offset);
1162 *data = 0;
1163 goto done;
1164 }
1165
1166 if (!x2apic(vlapic) && !mmio_access) {
1167 /*
1168 * XXX Generate GP fault for MSR accesses in xAPIC mode
1169 */
1170 VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in "
1171 "xAPIC mode", offset);
1172 *data = 0;
1173 goto done;
1174 }
1175
1176 if (offset > sizeof(*lapic)) {
1177 *data = 0;
1178 goto done;
1179 }
1180
1181 offset &= ~3;
1182 switch(offset)
1183 {
1184 case APIC_OFFSET_ID:
1185 *data = lapic->id;
1186 break;
1187 case APIC_OFFSET_VER:
1188 *data = lapic->version;
1189 break;
1190 case APIC_OFFSET_TPR:
1191 *data = lapic->tpr;
1192 break;
1193 case APIC_OFFSET_APR:
1194 *data = lapic->apr;
1195 break;
1196 case APIC_OFFSET_PPR:
1197 *data = lapic->ppr;
1198 break;
1199 case APIC_OFFSET_EOI:
1200 *data = lapic->eoi;
1201 break;
1202 case APIC_OFFSET_LDR:
1203 *data = lapic->ldr;
1204 break;
1205 case APIC_OFFSET_DFR:
1206 *data = lapic->dfr;
1207 break;
1208 case APIC_OFFSET_SVR:
1209 *data = lapic->svr;
1210 break;
1211 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1212 i = (offset - APIC_OFFSET_ISR0) >> 2;
1213 reg = &lapic->isr0;
1214 *data = *(reg + i);
1215 break;
1216 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1217 i = (offset - APIC_OFFSET_TMR0) >> 2;
1218 reg = &lapic->tmr0;
1219 *data = *(reg + i);
1220 break;
1221 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1222 i = (offset - APIC_OFFSET_IRR0) >> 2;
1223 reg = &lapic->irr0;
1224 *data = atomic_load_acq_int(reg + i);
1225 break;
1226 case APIC_OFFSET_ESR:
1227 *data = lapic->esr;
1228 break;
1229 case APIC_OFFSET_ICR_LOW:
1230 *data = lapic->icr_lo;
1231 if (x2apic(vlapic))
1232 *data |= (uint64_t)lapic->icr_hi << 32;
1233 break;
1234 case APIC_OFFSET_ICR_HI:
1235 *data = lapic->icr_hi;
1236 break;
1237 case APIC_OFFSET_CMCI_LVT:
1238 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1239 *data = vlapic_get_lvt(vlapic, offset);
1240#ifdef INVARIANTS
1241 reg = vlapic_get_lvtptr(vlapic, offset);
1242 KASSERT(*data == *reg, ("inconsistent lvt value at "
1243 "offset %#lx: %#lx/%#x", offset, *data, *reg));
1244#endif
1245 break;
1246 case APIC_OFFSET_TIMER_ICR:
1247 *data = lapic->icr_timer;
1248 break;
1249 case APIC_OFFSET_TIMER_CCR:
1250 *data = vlapic_get_ccr(vlapic);
1251 break;
1252 case APIC_OFFSET_TIMER_DCR:
1253 *data = lapic->dcr_timer;
1254 break;
1255 case APIC_OFFSET_SELF_IPI:
1256 /*
1257 * XXX generate a GP fault if vlapic is in x2apic mode
1258 */
1259 *data = 0;
1260 break;
1261 case APIC_OFFSET_RRR:
1262 default:
1263 *data = 0;
1264 break;
1265 }
1266done:
1267 VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1268 return 0;
1269}
1270
1271int
1272vlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1273 uint64_t data, bool *retu)
1274{
1275 struct LAPIC *lapic = vlapic->apic_page;
1276 uint32_t *regptr;
1277 int retval;
1278
1279 KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
1280 ("vlapic_write: invalid offset %#lx", offset));
1281
1282 VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx",
1283 offset, data);
1284
1285 if (offset > sizeof(*lapic))
1286 return (0);
1287
1288 /* Ignore MMIO accesses in x2APIC mode */
1289 if (x2apic(vlapic) && mmio_access) {
1290 VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx "
1291 "in x2APIC mode", data, offset);
1292 return (0);
1293 }
1294
1295 /*
1296 * XXX Generate GP fault for MSR accesses in xAPIC mode
1297 */
1298 if (!x2apic(vlapic) && !mmio_access) {
1299 VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx "
1300 "in xAPIC mode", data, offset);
1301 return (0);
1302 }
1303
1304 retval = 0;
1305 switch(offset)
1306 {
1307 case APIC_OFFSET_ID:
1308 lapic->id = data;
1309 vlapic_id_write_handler(vlapic);
1310 break;
1311 case APIC_OFFSET_TPR:
1312 lapic->tpr = data & 0xff;
1313 vlapic_update_ppr(vlapic);
1314 break;
1315 case APIC_OFFSET_EOI:
1316 vlapic_process_eoi(vlapic);
1317 break;
1318 case APIC_OFFSET_LDR:
1319 lapic->ldr = data;
1320 vlapic_ldr_write_handler(vlapic);
1321 break;
1322 case APIC_OFFSET_DFR:
1323 lapic->dfr = data;
1324 vlapic_dfr_write_handler(vlapic);
1325 break;
1326 case APIC_OFFSET_SVR:
1327 lapic->svr = data;
1328 vlapic_svr_write_handler(vlapic);
1329 break;
1330 case APIC_OFFSET_ICR_LOW:
1331 lapic->icr_lo = data;
1332 if (x2apic(vlapic))
1333 lapic->icr_hi = data >> 32;
1334 retval = vlapic_icrlo_write_handler(vlapic, retu);
1335 break;
1336 case APIC_OFFSET_ICR_HI:
1337 lapic->icr_hi = data;
1338 break;
1339 case APIC_OFFSET_CMCI_LVT:
1340 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1341 regptr = vlapic_get_lvtptr(vlapic, offset);
1342 *regptr = data;
1343 vlapic_lvt_write_handler(vlapic, offset);
1344 break;
1345 case APIC_OFFSET_TIMER_ICR:
1346 lapic->icr_timer = data;
1347 vlapic_icrtmr_write_handler(vlapic);
1348 break;
1349
1350 case APIC_OFFSET_TIMER_DCR:
1351 lapic->dcr_timer = data;
1352 vlapic_dcr_write_handler(vlapic);
1353 break;
1354
1355 case APIC_OFFSET_ESR:
1356 vlapic_esr_write_handler(vlapic);
1357 break;
1358
1359 case APIC_OFFSET_SELF_IPI:
1360 if (x2apic(vlapic))
1361 vlapic_self_ipi_handler(vlapic, data);
1362 break;
1363
1364 case APIC_OFFSET_VER:
1365 case APIC_OFFSET_APR:
1366 case APIC_OFFSET_PPR:
1367 case APIC_OFFSET_RRR:
1368 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1369 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1370 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1371 case APIC_OFFSET_TIMER_CCR:
1372 default:
1373 // Read only.
1374 break;
1375 }
1376
1377 return (retval);
1378}
1379
1380static void
1381vlapic_reset(struct vlapic *vlapic)
1382{
1383 struct LAPIC *lapic;
1384
1385 lapic = vlapic->apic_page;
1386 bzero(lapic, sizeof(struct LAPIC));
1387
1388 lapic->id = vlapic_get_id(vlapic);
1389 lapic->version = VLAPIC_VERSION;
1390 lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
1391 lapic->dfr = 0xffffffff;
1392 lapic->svr = APIC_SVR_VECTOR;
1393 vlapic_mask_lvts(vlapic);
1394 vlapic_reset_tmr(vlapic);
1395
1396 lapic->dcr_timer = 0;
1397 vlapic_dcr_write_handler(vlapic);
1398
1399 if (vlapic->vcpuid == 0)
1400 vlapic->boot_state = BS_RUNNING; /* BSP */
1401 else
1402 vlapic->boot_state = BS_INIT; /* AP */
1403
1404 vlapic->svr_last = lapic->svr;
1405}
1406
1407void
1408vlapic_init(struct vlapic *vlapic)
1409{
1410 KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
1411 KASSERT(vlapic->vcpuid >= 0 && vlapic->vcpuid < VM_MAXCPU,
1412 ("vlapic_init: vcpuid is not initialized"));
1413 KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
1414 "initialized"));
1415
1416 /*
1417 * If the vlapic is configured in x2apic mode then it will be
1418 * accessed in the critical section via the MSR emulation code.
1419 *
1420 * Therefore the timer mutex must be a spinlock because blockable
1421 * mutexes cannot be acquired in a critical section.
1422 */
1423 mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1424 callout_init(&vlapic->callout, 1);
1425
1426 vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1427
1428 if (vlapic->vcpuid == 0)
1429 vlapic->msr_apicbase |= APICBASE_BSP;
1430
1431 vlapic_reset(vlapic);
1432}
1433
1434void
1435vlapic_cleanup(struct vlapic *vlapic)
1436{
1437
1438 callout_drain(&vlapic->callout);
1439}
1440
1441uint64_t
1442vlapic_get_apicbase(struct vlapic *vlapic)
1443{
1444
1445 return (vlapic->msr_apicbase);
1446}
1447
1448int
1449vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
1450{
1451
1452 if (vlapic->msr_apicbase != new) {
1453 VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx "
1454 "not supported", vlapic->msr_apicbase, new);
1455 return (-1);
1456 }
1457
1458 return (0);
1459}
1460
1461void
1462vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1463{
1464 struct vlapic *vlapic;
1465 struct LAPIC *lapic;
1466
1467 vlapic = vm_lapic(vm, vcpuid);
1468
1469 if (state == X2APIC_DISABLED)
1470 vlapic->msr_apicbase &= ~APICBASE_X2APIC;
1471 else
1472 vlapic->msr_apicbase |= APICBASE_X2APIC;
1473
1474 /*
1475 * Reset the local APIC registers whose values are mode-dependent.
1476 *
1477 * XXX this works because the APIC mode can be changed only at vcpu
1478 * initialization time.
1479 */
1480 lapic = vlapic->apic_page;
1481 lapic->id = vlapic_get_id(vlapic);
1482 if (x2apic(vlapic)) {
1483 lapic->ldr = x2apic_ldr(vlapic);
1484 lapic->dfr = 0;
1485 } else {
1486 lapic->ldr = 0;
1487 lapic->dfr = 0xffffffff;
1488 }
1489
1490 if (state == X2APIC_ENABLED) {
1491 if (vlapic->ops.enable_x2apic_mode)
1492 (*vlapic->ops.enable_x2apic_mode)(vlapic);
1493 }
1494}
1495
1496void
1497vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
1498 int delmode, int vec)
1499{
1500 bool lowprio;
1501 int vcpuid;
1502 cpuset_t dmask;
1503
1504 if (delmode != IOART_DELFIXED &&
1505 delmode != IOART_DELLOPRI &&
1506 delmode != IOART_DELEXINT) {
1507 VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
1508 return;
1509 }
1510 lowprio = (delmode == IOART_DELLOPRI);
1511
1512 /*
1513 * We don't provide any virtual interrupt redirection hardware so
1514 * all interrupts originating from the ioapic or MSI specify the
1515 * 'dest' in the legacy xAPIC format.
1516 */
1517 vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
1518
1519 while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1520 vcpuid--;
1521 CPU_CLR(vcpuid, &dmask);
1522 if (delmode == IOART_DELEXINT) {
1523 vm_inject_extint(vm, vcpuid);
1524 } else {
1525 lapic_set_intr(vm, vcpuid, vec, level);
1526 }
1527 }
1528}
1529
1530void
1531vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
1532{
1533 /*
1534 * Post an interrupt to the vcpu currently running on 'hostcpu'.
1535 *
1536 * This is done by leveraging features like Posted Interrupts (Intel)
1537 * Doorbell MSR (AMD AVIC) that avoid a VM exit.
1538 *
1539 * If neither of these features are available then fallback to
1540 * sending an IPI to 'hostcpu'.
1541 */
1542 if (vlapic->ops.post_intr)
1543 (*vlapic->ops.post_intr)(vlapic, hostcpu);
1544 else
1545 ipi_cpu(hostcpu, ipinum);
1546}
1547
1548bool
1549vlapic_enabled(struct vlapic *vlapic)
1550{
1551 struct LAPIC *lapic = vlapic->apic_page;
1552
1553 if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1554 (lapic->svr & APIC_SVR_ENABLE) != 0)
1555 return (true);
1556 else
1557 return (false);
1558}
1559
1560static void
1561vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
1562{
1563 struct LAPIC *lapic;
1564 uint32_t *tmrptr, mask;
1565 int idx;
1566
1567 lapic = vlapic->apic_page;
1568 tmrptr = &lapic->tmr0;
1569 idx = (vector / 32) * 4;
1570 mask = 1 << (vector % 32);
1571 if (level)
1572 tmrptr[idx] |= mask;
1573 else
1574 tmrptr[idx] &= ~mask;
1575
1576 if (vlapic->ops.set_tmr != NULL)
1577 (*vlapic->ops.set_tmr)(vlapic, vector, level);
1578}
1579
1580void
1581vlapic_reset_tmr(struct vlapic *vlapic)
1582{
1583 int vector;
1584
1585 VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
1586
1587 for (vector = 0; vector <= 255; vector++)
1588 vlapic_set_tmr(vlapic, vector, false);
1589}
1590
1591void
1592vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
1593 int delmode, int vector)
1594{
1595 cpuset_t dmask;
1596 bool lowprio;
1597
1598 KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
1599
1600 /*
1601 * A level trigger is valid only for fixed and lowprio delivery modes.
1602 */
1603 if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
1604 VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
1605 "delivery-mode %d", delmode);
1606 return;
1607 }
1608
1609 lowprio = (delmode == APIC_DELMODE_LOWPRIO);
1610 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
1611
1612 if (!CPU_ISSET(vlapic->vcpuid, &dmask))
1613 return;
1614
1615 VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
1616 vlapic_set_tmr(vlapic, vector, true);
1617}
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 268953 2014-07-21 19:08:02Z jhb $
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/io/vlapic.c 268953 2014-07-21 19:08:02Z jhb $");
31
32#include <sys/param.h>
33#include <sys/lock.h>
34#include <sys/kernel.h>
35#include <sys/malloc.h>
36#include <sys/mutex.h>
37#include <sys/systm.h>
38#include <sys/smp.h>
39
40#include <x86/specialreg.h>
41#include <x86/apicreg.h>
42
43#include <machine/clock.h>
44#include <machine/smp.h>
45
46#include <machine/vmm.h>
47
48#include "vmm_ipi.h"
49#include "vmm_lapic.h"
50#include "vmm_ktr.h"
51#include "vmm_stat.h"
52
53#include "vlapic.h"
54#include "vlapic_priv.h"
55#include "vioapic.h"
56
57#define PRIO(x) ((x) >> 4)
58
59#define VLAPIC_VERSION (16)
60
61#define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
62
63/*
64 * The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
65 * vlapic_callout_handler() and vcpu accesses to:
66 * - timer_freq_bt, timer_period_bt, timer_fire_bt
67 * - timer LVT register
68 */
69#define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx))
70#define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx))
71#define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx))
72
73/*
74 * APIC timer frequency:
75 * - arbitrary but chosen to be in the ballpark of contemporary hardware.
76 * - power-of-two to avoid loss of precision when converted to a bintime.
77 */
78#define VLAPIC_BUS_FREQ (128 * 1024 * 1024)
79
80static __inline uint32_t
81vlapic_get_id(struct vlapic *vlapic)
82{
83
84 if (x2apic(vlapic))
85 return (vlapic->vcpuid);
86 else
87 return (vlapic->vcpuid << 24);
88}
89
90static uint32_t
91x2apic_ldr(struct vlapic *vlapic)
92{
93 int apicid;
94 uint32_t ldr;
95
96 apicid = vlapic_get_id(vlapic);
97 ldr = 1 << (apicid & 0xf);
98 ldr |= (apicid & 0xffff0) << 12;
99 return (ldr);
100}
101
102void
103vlapic_dfr_write_handler(struct vlapic *vlapic)
104{
105 struct LAPIC *lapic;
106
107 lapic = vlapic->apic_page;
108 if (x2apic(vlapic)) {
109 VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
110 lapic->dfr);
111 lapic->dfr = 0;
112 return;
113 }
114
115 lapic->dfr &= APIC_DFR_MODEL_MASK;
116 lapic->dfr |= APIC_DFR_RESERVED;
117
118 if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
119 VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
120 else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
121 VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
122 else
123 VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
124}
125
126void
127vlapic_ldr_write_handler(struct vlapic *vlapic)
128{
129 struct LAPIC *lapic;
130
131 lapic = vlapic->apic_page;
132
133 /* LDR is read-only in x2apic mode */
134 if (x2apic(vlapic)) {
135 VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
136 lapic->ldr);
137 lapic->ldr = x2apic_ldr(vlapic);
138 } else {
139 lapic->ldr &= ~APIC_LDR_RESERVED;
140 VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
141 }
142}
143
144void
145vlapic_id_write_handler(struct vlapic *vlapic)
146{
147 struct LAPIC *lapic;
148
149 /*
150 * We don't allow the ID register to be modified so reset it back to
151 * its default value.
152 */
153 lapic = vlapic->apic_page;
154 lapic->id = vlapic_get_id(vlapic);
155}
156
157static int
158vlapic_timer_divisor(uint32_t dcr)
159{
160 switch (dcr & 0xB) {
161 case APIC_TDCR_1:
162 return (1);
163 case APIC_TDCR_2:
164 return (2);
165 case APIC_TDCR_4:
166 return (4);
167 case APIC_TDCR_8:
168 return (8);
169 case APIC_TDCR_16:
170 return (16);
171 case APIC_TDCR_32:
172 return (32);
173 case APIC_TDCR_64:
174 return (64);
175 case APIC_TDCR_128:
176 return (128);
177 default:
178 panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
179 }
180}
181
182#if 0
183static inline void
184vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
185{
186 printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
187 *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
188 *lvt & APIC_LVTT_M);
189}
190#endif
191
192static uint32_t
193vlapic_get_ccr(struct vlapic *vlapic)
194{
195 struct bintime bt_now, bt_rem;
196 struct LAPIC *lapic;
197 uint32_t ccr;
198
199 ccr = 0;
200 lapic = vlapic->apic_page;
201
202 VLAPIC_TIMER_LOCK(vlapic);
203 if (callout_active(&vlapic->callout)) {
204 /*
205 * If the timer is scheduled to expire in the future then
206 * compute the value of 'ccr' based on the remaining time.
207 */
208 binuptime(&bt_now);
209 if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
210 bt_rem = vlapic->timer_fire_bt;
211 bintime_sub(&bt_rem, &bt_now);
212 ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
213 ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
214 }
215 }
216 KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
217 "icr_timer is %#x", ccr, lapic->icr_timer));
218 VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
219 ccr, lapic->icr_timer);
220 VLAPIC_TIMER_UNLOCK(vlapic);
221 return (ccr);
222}
223
224void
225vlapic_dcr_write_handler(struct vlapic *vlapic)
226{
227 struct LAPIC *lapic;
228 int divisor;
229
230 lapic = vlapic->apic_page;
231 VLAPIC_TIMER_LOCK(vlapic);
232
233 divisor = vlapic_timer_divisor(lapic->dcr_timer);
234 VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
235 lapic->dcr_timer, divisor);
236
237 /*
238 * Update the timer frequency and the timer period.
239 *
240 * XXX changes to the frequency divider will not take effect until
241 * the timer is reloaded.
242 */
243 FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
244 vlapic->timer_period_bt = vlapic->timer_freq_bt;
245 bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
246
247 VLAPIC_TIMER_UNLOCK(vlapic);
248}
249
250void
251vlapic_esr_write_handler(struct vlapic *vlapic)
252{
253 struct LAPIC *lapic;
254
255 lapic = vlapic->apic_page;
256 lapic->esr = vlapic->esr_pending;
257 vlapic->esr_pending = 0;
258}
259
260int
261vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
262{
263 struct LAPIC *lapic;
264 uint32_t *irrptr, *tmrptr, mask;
265 int idx;
266
267 KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
268
269 lapic = vlapic->apic_page;
270 if (!(lapic->svr & APIC_SVR_ENABLE)) {
271 VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
272 "interrupt %d", vector);
273 return (0);
274 }
275
276 if (vector < 16) {
277 vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
278 VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
279 vector);
280 return (1);
281 }
282
283 if (vlapic->ops.set_intr_ready)
284 return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
285
286 idx = (vector / 32) * 4;
287 mask = 1 << (vector % 32);
288
289 irrptr = &lapic->irr0;
290 atomic_set_int(&irrptr[idx], mask);
291
292 /*
293 * Verify that the trigger-mode of the interrupt matches with
294 * the vlapic TMR registers.
295 */
296 tmrptr = &lapic->tmr0;
297 if ((tmrptr[idx] & mask) != (level ? mask : 0)) {
298 VLAPIC_CTR3(vlapic, "vlapic TMR[%d] is 0x%08x but "
299 "interrupt is %s-triggered", idx / 4, tmrptr[idx],
300 level ? "level" : "edge");
301 }
302
303 VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
304 return (1);
305}
306
307static __inline uint32_t *
308vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
309{
310 struct LAPIC *lapic = vlapic->apic_page;
311 int i;
312
313 switch (offset) {
314 case APIC_OFFSET_CMCI_LVT:
315 return (&lapic->lvt_cmci);
316 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
317 i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
318 return ((&lapic->lvt_timer) + i);;
319 default:
320 panic("vlapic_get_lvt: invalid LVT\n");
321 }
322}
323
324static __inline int
325lvt_off_to_idx(uint32_t offset)
326{
327 int index;
328
329 switch (offset) {
330 case APIC_OFFSET_CMCI_LVT:
331 index = APIC_LVT_CMCI;
332 break;
333 case APIC_OFFSET_TIMER_LVT:
334 index = APIC_LVT_TIMER;
335 break;
336 case APIC_OFFSET_THERM_LVT:
337 index = APIC_LVT_THERMAL;
338 break;
339 case APIC_OFFSET_PERF_LVT:
340 index = APIC_LVT_PMC;
341 break;
342 case APIC_OFFSET_LINT0_LVT:
343 index = APIC_LVT_LINT0;
344 break;
345 case APIC_OFFSET_LINT1_LVT:
346 index = APIC_LVT_LINT1;
347 break;
348 case APIC_OFFSET_ERROR_LVT:
349 index = APIC_LVT_ERROR;
350 break;
351 default:
352 index = -1;
353 break;
354 }
355 KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
356 "invalid lvt index %d for offset %#x", index, offset));
357
358 return (index);
359}
360
361static __inline uint32_t
362vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
363{
364 int idx;
365 uint32_t val;
366
367 idx = lvt_off_to_idx(offset);
368 val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
369 return (val);
370}
371
372void
373vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
374{
375 uint32_t *lvtptr, mask, val;
376 struct LAPIC *lapic;
377 int idx;
378
379 lapic = vlapic->apic_page;
380 lvtptr = vlapic_get_lvtptr(vlapic, offset);
381 val = *lvtptr;
382 idx = lvt_off_to_idx(offset);
383
384 if (!(lapic->svr & APIC_SVR_ENABLE))
385 val |= APIC_LVT_M;
386 mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
387 switch (offset) {
388 case APIC_OFFSET_TIMER_LVT:
389 mask |= APIC_LVTT_TM;
390 break;
391 case APIC_OFFSET_ERROR_LVT:
392 break;
393 case APIC_OFFSET_LINT0_LVT:
394 case APIC_OFFSET_LINT1_LVT:
395 mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
396 /* FALLTHROUGH */
397 default:
398 mask |= APIC_LVT_DM;
399 break;
400 }
401 val &= mask;
402 *lvtptr = val;
403 atomic_store_rel_32(&vlapic->lvt_last[idx], val);
404}
405
406static void
407vlapic_mask_lvts(struct vlapic *vlapic)
408{
409 struct LAPIC *lapic = vlapic->apic_page;
410
411 lapic->lvt_cmci |= APIC_LVT_M;
412 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
413
414 lapic->lvt_timer |= APIC_LVT_M;
415 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
416
417 lapic->lvt_thermal |= APIC_LVT_M;
418 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
419
420 lapic->lvt_pcint |= APIC_LVT_M;
421 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
422
423 lapic->lvt_lint0 |= APIC_LVT_M;
424 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
425
426 lapic->lvt_lint1 |= APIC_LVT_M;
427 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
428
429 lapic->lvt_error |= APIC_LVT_M;
430 vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
431}
432
433static int
434vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
435{
436 uint32_t vec, mode;
437
438 if (lvt & APIC_LVT_M)
439 return (0);
440
441 vec = lvt & APIC_LVT_VECTOR;
442 mode = lvt & APIC_LVT_DM;
443
444 switch (mode) {
445 case APIC_LVT_DM_FIXED:
446 if (vec < 16) {
447 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
448 return (0);
449 }
450 if (vlapic_set_intr_ready(vlapic, vec, false))
451 vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
452 break;
453 case APIC_LVT_DM_NMI:
454 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
455 break;
456 case APIC_LVT_DM_EXTINT:
457 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
458 break;
459 default:
460 // Other modes ignored
461 return (0);
462 }
463 return (1);
464}
465
466#if 1
467static void
468dump_isrvec_stk(struct vlapic *vlapic)
469{
470 int i;
471 uint32_t *isrptr;
472
473 isrptr = &vlapic->apic_page->isr0;
474 for (i = 0; i < 8; i++)
475 printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
476
477 for (i = 0; i <= vlapic->isrvec_stk_top; i++)
478 printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
479}
480#endif
481
482/*
483 * Algorithm adopted from section "Interrupt, Task and Processor Priority"
484 * in Intel Architecture Manual Vol 3a.
485 */
486static void
487vlapic_update_ppr(struct vlapic *vlapic)
488{
489 int isrvec, tpr, ppr;
490
491 /*
492 * Note that the value on the stack at index 0 is always 0.
493 *
494 * This is a placeholder for the value of ISRV when none of the
495 * bits is set in the ISRx registers.
496 */
497 isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
498 tpr = vlapic->apic_page->tpr;
499
500#if 1
501 {
502 int i, lastprio, curprio, vector, idx;
503 uint32_t *isrptr;
504
505 if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
506 panic("isrvec_stk is corrupted: %d", isrvec);
507
508 /*
509 * Make sure that the priority of the nested interrupts is
510 * always increasing.
511 */
512 lastprio = -1;
513 for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
514 curprio = PRIO(vlapic->isrvec_stk[i]);
515 if (curprio <= lastprio) {
516 dump_isrvec_stk(vlapic);
517 panic("isrvec_stk does not satisfy invariant");
518 }
519 lastprio = curprio;
520 }
521
522 /*
523 * Make sure that each bit set in the ISRx registers has a
524 * corresponding entry on the isrvec stack.
525 */
526 i = 1;
527 isrptr = &vlapic->apic_page->isr0;
528 for (vector = 0; vector < 256; vector++) {
529 idx = (vector / 32) * 4;
530 if (isrptr[idx] & (1 << (vector % 32))) {
531 if (i > vlapic->isrvec_stk_top ||
532 vlapic->isrvec_stk[i] != vector) {
533 dump_isrvec_stk(vlapic);
534 panic("ISR and isrvec_stk out of sync");
535 }
536 i++;
537 }
538 }
539 }
540#endif
541
542 if (PRIO(tpr) >= PRIO(isrvec))
543 ppr = tpr;
544 else
545 ppr = isrvec & 0xf0;
546
547 vlapic->apic_page->ppr = ppr;
548 VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
549}
550
551static void
552vlapic_process_eoi(struct vlapic *vlapic)
553{
554 struct LAPIC *lapic = vlapic->apic_page;
555 uint32_t *isrptr, *tmrptr;
556 int i, idx, bitpos, vector;
557
558 isrptr = &lapic->isr0;
559 tmrptr = &lapic->tmr0;
560
561 /*
562 * The x86 architecture reserves the the first 32 vectors for use
563 * by the processor.
564 */
565 for (i = 7; i > 0; i--) {
566 idx = i * 4;
567 bitpos = fls(isrptr[idx]);
568 if (bitpos-- != 0) {
569 if (vlapic->isrvec_stk_top <= 0) {
570 panic("invalid vlapic isrvec_stk_top %d",
571 vlapic->isrvec_stk_top);
572 }
573 isrptr[idx] &= ~(1 << bitpos);
574 VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
575 vlapic->isrvec_stk_top--;
576 vlapic_update_ppr(vlapic);
577 if ((tmrptr[idx] & (1 << bitpos)) != 0) {
578 vector = i * 32 + bitpos;
579 vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
580 vector);
581 }
582 return;
583 }
584 }
585}
586
587static __inline int
588vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
589{
590
591 return (lvt & mask);
592}
593
594static __inline int
595vlapic_periodic_timer(struct vlapic *vlapic)
596{
597 uint32_t lvt;
598
599 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
600
601 return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
602}
603
604static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
605
606void
607vlapic_set_error(struct vlapic *vlapic, uint32_t mask)
608{
609 uint32_t lvt;
610
611 vlapic->esr_pending |= mask;
612 if (vlapic->esr_firing)
613 return;
614 vlapic->esr_firing = 1;
615
616 // The error LVT always uses the fixed delivery mode.
617 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
618 if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
619 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1);
620 }
621 vlapic->esr_firing = 0;
622}
623
624static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
625
626static void
627vlapic_fire_timer(struct vlapic *vlapic)
628{
629 uint32_t lvt;
630
631 KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
632
633 // The timer LVT always uses the fixed delivery mode.
634 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
635 if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
636 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
637 }
638}
639
640static VMM_STAT(VLAPIC_INTR_CMC,
641 "corrected machine check interrupts generated by vlapic");
642
643void
644vlapic_fire_cmci(struct vlapic *vlapic)
645{
646 uint32_t lvt;
647
648 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
649 if (vlapic_fire_lvt(vlapic, lvt)) {
650 vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
651 }
652}
653
654static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
655 "lvts triggered");
656
657int
658vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
659{
660 uint32_t lvt;
661
662 if (vlapic_enabled(vlapic) == false) {
663 /*
664 * When the local APIC is global/hardware disabled,
665 * LINT[1:0] pins are configured as INTR and NMI pins,
666 * respectively.
667 */
668 switch (vector) {
669 case APIC_LVT_LINT0:
670 vm_inject_extint(vlapic->vm, vlapic->vcpuid);
671 break;
672 case APIC_LVT_LINT1:
673 vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
674 break;
675 default:
676 break;
677 }
678 return (0);
679 }
680
681 switch (vector) {
682 case APIC_LVT_LINT0:
683 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT);
684 break;
685 case APIC_LVT_LINT1:
686 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT);
687 break;
688 case APIC_LVT_TIMER:
689 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
690 lvt |= APIC_LVT_DM_FIXED;
691 break;
692 case APIC_LVT_ERROR:
693 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
694 lvt |= APIC_LVT_DM_FIXED;
695 break;
696 case APIC_LVT_PMC:
697 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT);
698 break;
699 case APIC_LVT_THERMAL:
700 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT);
701 break;
702 case APIC_LVT_CMCI:
703 lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
704 break;
705 default:
706 return (EINVAL);
707 }
708 if (vlapic_fire_lvt(vlapic, lvt)) {
709 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
710 LVTS_TRIGGERRED, vector, 1);
711 }
712 return (0);
713}
714
715static void
716vlapic_callout_handler(void *arg)
717{
718 struct vlapic *vlapic;
719 struct bintime bt, btnow;
720 sbintime_t rem_sbt;
721
722 vlapic = arg;
723
724 VLAPIC_TIMER_LOCK(vlapic);
725 if (callout_pending(&vlapic->callout)) /* callout was reset */
726 goto done;
727
728 if (!callout_active(&vlapic->callout)) /* callout was stopped */
729 goto done;
730
731 callout_deactivate(&vlapic->callout);
732
733 vlapic_fire_timer(vlapic);
734
735 if (vlapic_periodic_timer(vlapic)) {
736 binuptime(&btnow);
737 KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
738 ("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
739 btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
740 vlapic->timer_fire_bt.frac));
741
742 /*
743 * Compute the delta between when the timer was supposed to
744 * fire and the present time.
745 */
746 bt = btnow;
747 bintime_sub(&bt, &vlapic->timer_fire_bt);
748
749 rem_sbt = bttosbt(vlapic->timer_period_bt);
750 if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
751 /*
752 * Adjust the time until the next countdown downward
753 * to account for the lost time.
754 */
755 rem_sbt -= bttosbt(bt);
756 } else {
757 /*
758 * If the delta is greater than the timer period then
759 * just reset our time base instead of trying to catch
760 * up.
761 */
762 vlapic->timer_fire_bt = btnow;
763 VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
764 "usecs, period is %lu usecs - resetting time base",
765 bttosbt(bt) / SBT_1US,
766 bttosbt(vlapic->timer_period_bt) / SBT_1US);
767 }
768
769 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
770 callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
771 vlapic_callout_handler, vlapic, 0);
772 }
773done:
774 VLAPIC_TIMER_UNLOCK(vlapic);
775}
776
777void
778vlapic_icrtmr_write_handler(struct vlapic *vlapic)
779{
780 struct LAPIC *lapic;
781 sbintime_t sbt;
782 uint32_t icr_timer;
783
784 VLAPIC_TIMER_LOCK(vlapic);
785
786 lapic = vlapic->apic_page;
787 icr_timer = lapic->icr_timer;
788
789 vlapic->timer_period_bt = vlapic->timer_freq_bt;
790 bintime_mul(&vlapic->timer_period_bt, icr_timer);
791
792 if (icr_timer != 0) {
793 binuptime(&vlapic->timer_fire_bt);
794 bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
795
796 sbt = bttosbt(vlapic->timer_period_bt);
797 callout_reset_sbt(&vlapic->callout, sbt, 0,
798 vlapic_callout_handler, vlapic, 0);
799 } else
800 callout_stop(&vlapic->callout);
801
802 VLAPIC_TIMER_UNLOCK(vlapic);
803}
804
805/*
806 * This function populates 'dmask' with the set of vcpus that match the
807 * addressing specified by the (dest, phys, lowprio) tuple.
808 *
809 * 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
810 * or xAPIC (8-bit) destination field.
811 */
812static void
813vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
814 bool lowprio, bool x2apic_dest)
815{
816 struct vlapic *vlapic;
817 uint32_t dfr, ldr, ldest, cluster;
818 uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
819 cpuset_t amask;
820 int vcpuid;
821
822 if ((x2apic_dest && dest == 0xffffffff) ||
823 (!x2apic_dest && dest == 0xff)) {
824 /*
825 * Broadcast in both logical and physical modes.
826 */
827 *dmask = vm_active_cpus(vm);
828 return;
829 }
830
831 if (phys) {
832 /*
833 * Physical mode: destination is APIC ID.
834 */
835 CPU_ZERO(dmask);
836 vcpuid = vm_apicid2vcpuid(vm, dest);
837 if (vcpuid < VM_MAXCPU)
838 CPU_SET(vcpuid, dmask);
839 } else {
840 /*
841 * In the "Flat Model" the MDA is interpreted as an 8-bit wide
842 * bitmask. This model is only avilable in the xAPIC mode.
843 */
844 mda_flat_ldest = dest & 0xff;
845
846 /*
847 * In the "Cluster Model" the MDA is used to identify a
848 * specific cluster and a set of APICs in that cluster.
849 */
850 if (x2apic_dest) {
851 mda_cluster_id = dest >> 16;
852 mda_cluster_ldest = dest & 0xffff;
853 } else {
854 mda_cluster_id = (dest >> 4) & 0xf;
855 mda_cluster_ldest = dest & 0xf;
856 }
857
858 /*
859 * Logical mode: match each APIC that has a bit set
860 * in it's LDR that matches a bit in the ldest.
861 */
862 CPU_ZERO(dmask);
863 amask = vm_active_cpus(vm);
864 while ((vcpuid = CPU_FFS(&amask)) != 0) {
865 vcpuid--;
866 CPU_CLR(vcpuid, &amask);
867
868 vlapic = vm_lapic(vm, vcpuid);
869 dfr = vlapic->apic_page->dfr;
870 ldr = vlapic->apic_page->ldr;
871
872 if ((dfr & APIC_DFR_MODEL_MASK) ==
873 APIC_DFR_MODEL_FLAT) {
874 ldest = ldr >> 24;
875 mda_ldest = mda_flat_ldest;
876 } else if ((dfr & APIC_DFR_MODEL_MASK) ==
877 APIC_DFR_MODEL_CLUSTER) {
878 if (x2apic(vlapic)) {
879 cluster = ldr >> 16;
880 ldest = ldr & 0xffff;
881 } else {
882 cluster = ldr >> 28;
883 ldest = (ldr >> 24) & 0xf;
884 }
885 if (cluster != mda_cluster_id)
886 continue;
887 mda_ldest = mda_cluster_ldest;
888 } else {
889 /*
890 * Guest has configured a bad logical
891 * model for this vcpu - skip it.
892 */
893 VLAPIC_CTR1(vlapic, "vlapic has bad logical "
894 "model %x - cannot deliver interrupt", dfr);
895 continue;
896 }
897
898 if ((mda_ldest & ldest) != 0) {
899 CPU_SET(vcpuid, dmask);
900 if (lowprio)
901 break;
902 }
903 }
904 }
905}
906
907static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
908
909int
910vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
911{
912 int i;
913 bool phys;
914 cpuset_t dmask;
915 uint64_t icrval;
916 uint32_t dest, vec, mode;
917 struct vlapic *vlapic2;
918 struct vm_exit *vmexit;
919 struct LAPIC *lapic;
920
921 lapic = vlapic->apic_page;
922 lapic->icr_lo &= ~APIC_DELSTAT_PEND;
923 icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
924
925 if (x2apic(vlapic))
926 dest = icrval >> 32;
927 else
928 dest = icrval >> (32 + 24);
929 vec = icrval & APIC_VECTOR_MASK;
930 mode = icrval & APIC_DELMODE_MASK;
931
932 if (mode == APIC_DELMODE_FIXED && vec < 16) {
933 vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
934 VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
935 return (0);
936 }
937
938 VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
939
940 if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
941 switch (icrval & APIC_DEST_MASK) {
942 case APIC_DEST_DESTFLD:
943 phys = ((icrval & APIC_DESTMODE_LOG) == 0);
944 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
945 x2apic(vlapic));
946 break;
947 case APIC_DEST_SELF:
948 CPU_SETOF(vlapic->vcpuid, &dmask);
949 break;
950 case APIC_DEST_ALLISELF:
951 dmask = vm_active_cpus(vlapic->vm);
952 break;
953 case APIC_DEST_ALLESELF:
954 dmask = vm_active_cpus(vlapic->vm);
955 CPU_CLR(vlapic->vcpuid, &dmask);
956 break;
957 default:
958 CPU_ZERO(&dmask); /* satisfy gcc */
959 break;
960 }
961
962 while ((i = CPU_FFS(&dmask)) != 0) {
963 i--;
964 CPU_CLR(i, &dmask);
965 if (mode == APIC_DELMODE_FIXED) {
966 lapic_intr_edge(vlapic->vm, i, vec);
967 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
968 IPIS_SENT, i, 1);
969 VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
970 "to vcpuid %d", vec, i);
971 } else {
972 vm_inject_nmi(vlapic->vm, i);
973 VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
974 "to vcpuid %d", i);
975 }
976 }
977
978 return (0); /* handled completely in the kernel */
979 }
980
981 if (mode == APIC_DELMODE_INIT) {
982 if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
983 return (0);
984
985 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
986 vlapic2 = vm_lapic(vlapic->vm, dest);
987
988 /* move from INIT to waiting-for-SIPI state */
989 if (vlapic2->boot_state == BS_INIT) {
990 vlapic2->boot_state = BS_SIPI;
991 }
992
993 return (0);
994 }
995 }
996
997 if (mode == APIC_DELMODE_STARTUP) {
998 if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
999 vlapic2 = vm_lapic(vlapic->vm, dest);
1000
1001 /*
1002 * Ignore SIPIs in any state other than wait-for-SIPI
1003 */
1004 if (vlapic2->boot_state != BS_SIPI)
1005 return (0);
1006
1007 /*
1008 * XXX this assumes that the startup IPI always succeeds
1009 */
1010 vlapic2->boot_state = BS_RUNNING;
1011 vm_activate_cpu(vlapic2->vm, dest);
1012
1013 *retu = true;
1014 vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
1015 vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
1016 vmexit->u.spinup_ap.vcpu = dest;
1017 vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
1018
1019 return (0);
1020 }
1021 }
1022
1023 /*
1024 * This will cause a return to userland.
1025 */
1026 return (1);
1027}
1028
1029void
1030vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val)
1031{
1032 int vec;
1033
1034 KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode"));
1035
1036 vec = val & 0xff;
1037 lapic_intr_edge(vlapic->vm, vlapic->vcpuid, vec);
1038 vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, IPIS_SENT,
1039 vlapic->vcpuid, 1);
1040 VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec);
1041}
1042
1043int
1044vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
1045{
1046 struct LAPIC *lapic = vlapic->apic_page;
1047 int idx, i, bitpos, vector;
1048 uint32_t *irrptr, val;
1049
1050 if (vlapic->ops.pending_intr)
1051 return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
1052
1053 irrptr = &lapic->irr0;
1054
1055 /*
1056 * The x86 architecture reserves the the first 32 vectors for use
1057 * by the processor.
1058 */
1059 for (i = 7; i > 0; i--) {
1060 idx = i * 4;
1061 val = atomic_load_acq_int(&irrptr[idx]);
1062 bitpos = fls(val);
1063 if (bitpos != 0) {
1064 vector = i * 32 + (bitpos - 1);
1065 if (PRIO(vector) > PRIO(lapic->ppr)) {
1066 VLAPIC_CTR1(vlapic, "pending intr %d", vector);
1067 if (vecptr != NULL)
1068 *vecptr = vector;
1069 return (1);
1070 } else
1071 break;
1072 }
1073 }
1074 return (0);
1075}
1076
1077void
1078vlapic_intr_accepted(struct vlapic *vlapic, int vector)
1079{
1080 struct LAPIC *lapic = vlapic->apic_page;
1081 uint32_t *irrptr, *isrptr;
1082 int idx, stk_top;
1083
1084 if (vlapic->ops.intr_accepted)
1085 return ((*vlapic->ops.intr_accepted)(vlapic, vector));
1086
1087 /*
1088 * clear the ready bit for vector being accepted in irr
1089 * and set the vector as in service in isr.
1090 */
1091 idx = (vector / 32) * 4;
1092
1093 irrptr = &lapic->irr0;
1094 atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
1095 VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
1096
1097 isrptr = &lapic->isr0;
1098 isrptr[idx] |= 1 << (vector % 32);
1099 VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
1100
1101 /*
1102 * Update the PPR
1103 */
1104 vlapic->isrvec_stk_top++;
1105
1106 stk_top = vlapic->isrvec_stk_top;
1107 if (stk_top >= ISRVEC_STK_SIZE)
1108 panic("isrvec_stk_top overflow %d", stk_top);
1109
1110 vlapic->isrvec_stk[stk_top] = vector;
1111 vlapic_update_ppr(vlapic);
1112}
1113
1114void
1115vlapic_svr_write_handler(struct vlapic *vlapic)
1116{
1117 struct LAPIC *lapic;
1118 uint32_t old, new, changed;
1119
1120 lapic = vlapic->apic_page;
1121
1122 new = lapic->svr;
1123 old = vlapic->svr_last;
1124 vlapic->svr_last = new;
1125
1126 changed = old ^ new;
1127 if ((changed & APIC_SVR_ENABLE) != 0) {
1128 if ((new & APIC_SVR_ENABLE) == 0) {
1129 /*
1130 * The apic is now disabled so stop the apic timer
1131 * and mask all the LVT entries.
1132 */
1133 VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
1134 VLAPIC_TIMER_LOCK(vlapic);
1135 callout_stop(&vlapic->callout);
1136 VLAPIC_TIMER_UNLOCK(vlapic);
1137 vlapic_mask_lvts(vlapic);
1138 } else {
1139 /*
1140 * The apic is now enabled so restart the apic timer
1141 * if it is configured in periodic mode.
1142 */
1143 VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
1144 if (vlapic_periodic_timer(vlapic))
1145 vlapic_icrtmr_write_handler(vlapic);
1146 }
1147 }
1148}
1149
1150int
1151vlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1152 uint64_t *data, bool *retu)
1153{
1154 struct LAPIC *lapic = vlapic->apic_page;
1155 uint32_t *reg;
1156 int i;
1157
1158 /* Ignore MMIO accesses in x2APIC mode */
1159 if (x2apic(vlapic) && mmio_access) {
1160 VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode",
1161 offset);
1162 *data = 0;
1163 goto done;
1164 }
1165
1166 if (!x2apic(vlapic) && !mmio_access) {
1167 /*
1168 * XXX Generate GP fault for MSR accesses in xAPIC mode
1169 */
1170 VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in "
1171 "xAPIC mode", offset);
1172 *data = 0;
1173 goto done;
1174 }
1175
1176 if (offset > sizeof(*lapic)) {
1177 *data = 0;
1178 goto done;
1179 }
1180
1181 offset &= ~3;
1182 switch(offset)
1183 {
1184 case APIC_OFFSET_ID:
1185 *data = lapic->id;
1186 break;
1187 case APIC_OFFSET_VER:
1188 *data = lapic->version;
1189 break;
1190 case APIC_OFFSET_TPR:
1191 *data = lapic->tpr;
1192 break;
1193 case APIC_OFFSET_APR:
1194 *data = lapic->apr;
1195 break;
1196 case APIC_OFFSET_PPR:
1197 *data = lapic->ppr;
1198 break;
1199 case APIC_OFFSET_EOI:
1200 *data = lapic->eoi;
1201 break;
1202 case APIC_OFFSET_LDR:
1203 *data = lapic->ldr;
1204 break;
1205 case APIC_OFFSET_DFR:
1206 *data = lapic->dfr;
1207 break;
1208 case APIC_OFFSET_SVR:
1209 *data = lapic->svr;
1210 break;
1211 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1212 i = (offset - APIC_OFFSET_ISR0) >> 2;
1213 reg = &lapic->isr0;
1214 *data = *(reg + i);
1215 break;
1216 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1217 i = (offset - APIC_OFFSET_TMR0) >> 2;
1218 reg = &lapic->tmr0;
1219 *data = *(reg + i);
1220 break;
1221 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1222 i = (offset - APIC_OFFSET_IRR0) >> 2;
1223 reg = &lapic->irr0;
1224 *data = atomic_load_acq_int(reg + i);
1225 break;
1226 case APIC_OFFSET_ESR:
1227 *data = lapic->esr;
1228 break;
1229 case APIC_OFFSET_ICR_LOW:
1230 *data = lapic->icr_lo;
1231 if (x2apic(vlapic))
1232 *data |= (uint64_t)lapic->icr_hi << 32;
1233 break;
1234 case APIC_OFFSET_ICR_HI:
1235 *data = lapic->icr_hi;
1236 break;
1237 case APIC_OFFSET_CMCI_LVT:
1238 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1239 *data = vlapic_get_lvt(vlapic, offset);
1240#ifdef INVARIANTS
1241 reg = vlapic_get_lvtptr(vlapic, offset);
1242 KASSERT(*data == *reg, ("inconsistent lvt value at "
1243 "offset %#lx: %#lx/%#x", offset, *data, *reg));
1244#endif
1245 break;
1246 case APIC_OFFSET_TIMER_ICR:
1247 *data = lapic->icr_timer;
1248 break;
1249 case APIC_OFFSET_TIMER_CCR:
1250 *data = vlapic_get_ccr(vlapic);
1251 break;
1252 case APIC_OFFSET_TIMER_DCR:
1253 *data = lapic->dcr_timer;
1254 break;
1255 case APIC_OFFSET_SELF_IPI:
1256 /*
1257 * XXX generate a GP fault if vlapic is in x2apic mode
1258 */
1259 *data = 0;
1260 break;
1261 case APIC_OFFSET_RRR:
1262 default:
1263 *data = 0;
1264 break;
1265 }
1266done:
1267 VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
1268 return 0;
1269}
1270
1271int
1272vlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset,
1273 uint64_t data, bool *retu)
1274{
1275 struct LAPIC *lapic = vlapic->apic_page;
1276 uint32_t *regptr;
1277 int retval;
1278
1279 KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
1280 ("vlapic_write: invalid offset %#lx", offset));
1281
1282 VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx",
1283 offset, data);
1284
1285 if (offset > sizeof(*lapic))
1286 return (0);
1287
1288 /* Ignore MMIO accesses in x2APIC mode */
1289 if (x2apic(vlapic) && mmio_access) {
1290 VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx "
1291 "in x2APIC mode", data, offset);
1292 return (0);
1293 }
1294
1295 /*
1296 * XXX Generate GP fault for MSR accesses in xAPIC mode
1297 */
1298 if (!x2apic(vlapic) && !mmio_access) {
1299 VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx "
1300 "in xAPIC mode", data, offset);
1301 return (0);
1302 }
1303
1304 retval = 0;
1305 switch(offset)
1306 {
1307 case APIC_OFFSET_ID:
1308 lapic->id = data;
1309 vlapic_id_write_handler(vlapic);
1310 break;
1311 case APIC_OFFSET_TPR:
1312 lapic->tpr = data & 0xff;
1313 vlapic_update_ppr(vlapic);
1314 break;
1315 case APIC_OFFSET_EOI:
1316 vlapic_process_eoi(vlapic);
1317 break;
1318 case APIC_OFFSET_LDR:
1319 lapic->ldr = data;
1320 vlapic_ldr_write_handler(vlapic);
1321 break;
1322 case APIC_OFFSET_DFR:
1323 lapic->dfr = data;
1324 vlapic_dfr_write_handler(vlapic);
1325 break;
1326 case APIC_OFFSET_SVR:
1327 lapic->svr = data;
1328 vlapic_svr_write_handler(vlapic);
1329 break;
1330 case APIC_OFFSET_ICR_LOW:
1331 lapic->icr_lo = data;
1332 if (x2apic(vlapic))
1333 lapic->icr_hi = data >> 32;
1334 retval = vlapic_icrlo_write_handler(vlapic, retu);
1335 break;
1336 case APIC_OFFSET_ICR_HI:
1337 lapic->icr_hi = data;
1338 break;
1339 case APIC_OFFSET_CMCI_LVT:
1340 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1341 regptr = vlapic_get_lvtptr(vlapic, offset);
1342 *regptr = data;
1343 vlapic_lvt_write_handler(vlapic, offset);
1344 break;
1345 case APIC_OFFSET_TIMER_ICR:
1346 lapic->icr_timer = data;
1347 vlapic_icrtmr_write_handler(vlapic);
1348 break;
1349
1350 case APIC_OFFSET_TIMER_DCR:
1351 lapic->dcr_timer = data;
1352 vlapic_dcr_write_handler(vlapic);
1353 break;
1354
1355 case APIC_OFFSET_ESR:
1356 vlapic_esr_write_handler(vlapic);
1357 break;
1358
1359 case APIC_OFFSET_SELF_IPI:
1360 if (x2apic(vlapic))
1361 vlapic_self_ipi_handler(vlapic, data);
1362 break;
1363
1364 case APIC_OFFSET_VER:
1365 case APIC_OFFSET_APR:
1366 case APIC_OFFSET_PPR:
1367 case APIC_OFFSET_RRR:
1368 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1369 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1370 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1371 case APIC_OFFSET_TIMER_CCR:
1372 default:
1373 // Read only.
1374 break;
1375 }
1376
1377 return (retval);
1378}
1379
1380static void
1381vlapic_reset(struct vlapic *vlapic)
1382{
1383 struct LAPIC *lapic;
1384
1385 lapic = vlapic->apic_page;
1386 bzero(lapic, sizeof(struct LAPIC));
1387
1388 lapic->id = vlapic_get_id(vlapic);
1389 lapic->version = VLAPIC_VERSION;
1390 lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
1391 lapic->dfr = 0xffffffff;
1392 lapic->svr = APIC_SVR_VECTOR;
1393 vlapic_mask_lvts(vlapic);
1394 vlapic_reset_tmr(vlapic);
1395
1396 lapic->dcr_timer = 0;
1397 vlapic_dcr_write_handler(vlapic);
1398
1399 if (vlapic->vcpuid == 0)
1400 vlapic->boot_state = BS_RUNNING; /* BSP */
1401 else
1402 vlapic->boot_state = BS_INIT; /* AP */
1403
1404 vlapic->svr_last = lapic->svr;
1405}
1406
1407void
1408vlapic_init(struct vlapic *vlapic)
1409{
1410 KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
1411 KASSERT(vlapic->vcpuid >= 0 && vlapic->vcpuid < VM_MAXCPU,
1412 ("vlapic_init: vcpuid is not initialized"));
1413 KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
1414 "initialized"));
1415
1416 /*
1417 * If the vlapic is configured in x2apic mode then it will be
1418 * accessed in the critical section via the MSR emulation code.
1419 *
1420 * Therefore the timer mutex must be a spinlock because blockable
1421 * mutexes cannot be acquired in a critical section.
1422 */
1423 mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
1424 callout_init(&vlapic->callout, 1);
1425
1426 vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
1427
1428 if (vlapic->vcpuid == 0)
1429 vlapic->msr_apicbase |= APICBASE_BSP;
1430
1431 vlapic_reset(vlapic);
1432}
1433
1434void
1435vlapic_cleanup(struct vlapic *vlapic)
1436{
1437
1438 callout_drain(&vlapic->callout);
1439}
1440
1441uint64_t
1442vlapic_get_apicbase(struct vlapic *vlapic)
1443{
1444
1445 return (vlapic->msr_apicbase);
1446}
1447
1448int
1449vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
1450{
1451
1452 if (vlapic->msr_apicbase != new) {
1453 VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx "
1454 "not supported", vlapic->msr_apicbase, new);
1455 return (-1);
1456 }
1457
1458 return (0);
1459}
1460
1461void
1462vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
1463{
1464 struct vlapic *vlapic;
1465 struct LAPIC *lapic;
1466
1467 vlapic = vm_lapic(vm, vcpuid);
1468
1469 if (state == X2APIC_DISABLED)
1470 vlapic->msr_apicbase &= ~APICBASE_X2APIC;
1471 else
1472 vlapic->msr_apicbase |= APICBASE_X2APIC;
1473
1474 /*
1475 * Reset the local APIC registers whose values are mode-dependent.
1476 *
1477 * XXX this works because the APIC mode can be changed only at vcpu
1478 * initialization time.
1479 */
1480 lapic = vlapic->apic_page;
1481 lapic->id = vlapic_get_id(vlapic);
1482 if (x2apic(vlapic)) {
1483 lapic->ldr = x2apic_ldr(vlapic);
1484 lapic->dfr = 0;
1485 } else {
1486 lapic->ldr = 0;
1487 lapic->dfr = 0xffffffff;
1488 }
1489
1490 if (state == X2APIC_ENABLED) {
1491 if (vlapic->ops.enable_x2apic_mode)
1492 (*vlapic->ops.enable_x2apic_mode)(vlapic);
1493 }
1494}
1495
1496void
1497vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
1498 int delmode, int vec)
1499{
1500 bool lowprio;
1501 int vcpuid;
1502 cpuset_t dmask;
1503
1504 if (delmode != IOART_DELFIXED &&
1505 delmode != IOART_DELLOPRI &&
1506 delmode != IOART_DELEXINT) {
1507 VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
1508 return;
1509 }
1510 lowprio = (delmode == IOART_DELLOPRI);
1511
1512 /*
1513 * We don't provide any virtual interrupt redirection hardware so
1514 * all interrupts originating from the ioapic or MSI specify the
1515 * 'dest' in the legacy xAPIC format.
1516 */
1517 vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
1518
1519 while ((vcpuid = CPU_FFS(&dmask)) != 0) {
1520 vcpuid--;
1521 CPU_CLR(vcpuid, &dmask);
1522 if (delmode == IOART_DELEXINT) {
1523 vm_inject_extint(vm, vcpuid);
1524 } else {
1525 lapic_set_intr(vm, vcpuid, vec, level);
1526 }
1527 }
1528}
1529
1530void
1531vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
1532{
1533 /*
1534 * Post an interrupt to the vcpu currently running on 'hostcpu'.
1535 *
1536 * This is done by leveraging features like Posted Interrupts (Intel)
1537 * Doorbell MSR (AMD AVIC) that avoid a VM exit.
1538 *
1539 * If neither of these features are available then fallback to
1540 * sending an IPI to 'hostcpu'.
1541 */
1542 if (vlapic->ops.post_intr)
1543 (*vlapic->ops.post_intr)(vlapic, hostcpu);
1544 else
1545 ipi_cpu(hostcpu, ipinum);
1546}
1547
1548bool
1549vlapic_enabled(struct vlapic *vlapic)
1550{
1551 struct LAPIC *lapic = vlapic->apic_page;
1552
1553 if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
1554 (lapic->svr & APIC_SVR_ENABLE) != 0)
1555 return (true);
1556 else
1557 return (false);
1558}
1559
1560static void
1561vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
1562{
1563 struct LAPIC *lapic;
1564 uint32_t *tmrptr, mask;
1565 int idx;
1566
1567 lapic = vlapic->apic_page;
1568 tmrptr = &lapic->tmr0;
1569 idx = (vector / 32) * 4;
1570 mask = 1 << (vector % 32);
1571 if (level)
1572 tmrptr[idx] |= mask;
1573 else
1574 tmrptr[idx] &= ~mask;
1575
1576 if (vlapic->ops.set_tmr != NULL)
1577 (*vlapic->ops.set_tmr)(vlapic, vector, level);
1578}
1579
1580void
1581vlapic_reset_tmr(struct vlapic *vlapic)
1582{
1583 int vector;
1584
1585 VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
1586
1587 for (vector = 0; vector <= 255; vector++)
1588 vlapic_set_tmr(vlapic, vector, false);
1589}
1590
1591void
1592vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
1593 int delmode, int vector)
1594{
1595 cpuset_t dmask;
1596 bool lowprio;
1597
1598 KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
1599
1600 /*
1601 * A level trigger is valid only for fixed and lowprio delivery modes.
1602 */
1603 if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
1604 VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
1605 "delivery-mode %d", delmode);
1606 return;
1607 }
1608
1609 lowprio = (delmode == APIC_DELMODE_LOWPRIO);
1610 vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
1611
1612 if (!CPU_ISSET(vlapic->vcpuid, &dmask))
1613 return;
1614
1615 VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
1616 vlapic_set_tmr(vlapic, vector, true);
1617}