1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17#include <linux/kvm_types.h> 18#include <linux/hashtable.h> 19#include <linux/amd-iommu.h> 20#include <linux/kvm_host.h> 21 22#include <asm/irq_remapping.h> 23 24#include "trace.h" 25#include "lapic.h" 26#include "x86.h" 27#include "irq.h" 28#include "svm.h" 29 30/* 31 * Encode the arbitrary VM ID and the vCPU's default APIC ID, i.e the vCPU ID, 32 * into the GATag so that KVM can retrieve the correct vCPU from a GALog entry 33 * if an interrupt can't be delivered, e.g. because the vCPU isn't running. 34 * 35 * For the vCPU ID, use however many bits are currently allowed for the max 36 * guest physical APIC ID (limited by the size of the physical ID table), and 37 * use whatever bits remain to assign arbitrary AVIC IDs to VMs. Note, the 38 * size of the GATag is defined by hardware (32 bits), but is an opaque value 39 * as far as hardware is concerned. 40 */ 41#define AVIC_VCPU_ID_MASK AVIC_PHYSICAL_MAX_INDEX_MASK 42 43#define AVIC_VM_ID_SHIFT HWEIGHT32(AVIC_PHYSICAL_MAX_INDEX_MASK) 44#define AVIC_VM_ID_MASK (GENMASK(31, AVIC_VM_ID_SHIFT) >> AVIC_VM_ID_SHIFT) 45 46#define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VM_ID_SHIFT) & AVIC_VM_ID_MASK) 47#define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) 48 49#define __AVIC_GATAG(vm_id, vcpu_id) ((((vm_id) & AVIC_VM_ID_MASK) << AVIC_VM_ID_SHIFT) | \ 50 ((vcpu_id) & AVIC_VCPU_ID_MASK)) 51#define AVIC_GATAG(vm_id, vcpu_id) \ 52({ \ 53 u32 ga_tag = __AVIC_GATAG(vm_id, vcpu_id); \ 54 \ 55 WARN_ON_ONCE(AVIC_GATAG_TO_VCPUID(ga_tag) != (vcpu_id)); \ 56 WARN_ON_ONCE(AVIC_GATAG_TO_VMID(ga_tag) != (vm_id)); \ 57 ga_tag; \ 58}) 59 60static_assert(__AVIC_GATAG(AVIC_VM_ID_MASK, AVIC_VCPU_ID_MASK) == -1u); 61 62static bool force_avic; 63module_param_unsafe(force_avic, bool, 0444); 64 65/* Note: 66 * This hash table is used to map VM_ID to a struct kvm_svm, 67 * when handling AMD IOMMU GALOG notification to schedule in 68 * a particular vCPU. 69 */ 70#define SVM_VM_DATA_HASH_BITS 8 71static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS); 72static u32 next_vm_id = 0; 73static bool next_vm_id_wrapped = 0; 74static DEFINE_SPINLOCK(svm_vm_data_hash_lock); 75bool x2avic_enabled; 76 77/* 78 * This is a wrapper of struct amd_iommu_ir_data. 79 */ 80struct amd_svm_iommu_ir { 81 struct list_head node; /* Used by SVM for per-vcpu ir_list */ 82 void *data; /* Storing pointer to struct amd_ir_data */ 83}; 84 85static void avic_activate_vmcb(struct vcpu_svm *svm) 86{ 87 struct vmcb *vmcb = svm->vmcb01.ptr; 88 89 vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK); 90 vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK; 91 92 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 93 94 /* 95 * Note: KVM supports hybrid-AVIC mode, where KVM emulates x2APIC MSR 96 * accesses, while interrupt injection to a running vCPU can be 97 * achieved using AVIC doorbell. KVM disables the APIC access page 98 * (deletes the memslot) if any vCPU has x2APIC enabled, thus enabling 99 * AVIC in hybrid mode activates only the doorbell mechanism. 100 */ 101 if (x2avic_enabled && apic_x2apic_mode(svm->vcpu.arch.apic)) { 102 vmcb->control.int_ctl |= X2APIC_MODE_MASK; 103 vmcb->control.avic_physical_id |= X2AVIC_MAX_PHYSICAL_ID; 104 /* Disabling MSR intercept for x2APIC registers */ 105 svm_set_x2apic_msr_interception(svm, false); 106 } else { 107 /* 108 * Flush the TLB, the guest may have inserted a non-APIC 109 * mapping into the TLB while AVIC was disabled. 110 */ 111 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, &svm->vcpu); 112 113 /* For xAVIC and hybrid-xAVIC modes */ 114 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID; 115 /* Enabling MSR intercept for x2APIC registers */ 116 svm_set_x2apic_msr_interception(svm, true); 117 } 118} 119 120static void avic_deactivate_vmcb(struct vcpu_svm *svm) 121{ 122 struct vmcb *vmcb = svm->vmcb01.ptr; 123 124 vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK); 125 vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK; 126 127 /* 128 * If running nested and the guest uses its own MSR bitmap, there 129 * is no need to update L0's msr bitmap 130 */ 131 if (is_guest_mode(&svm->vcpu) && 132 vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)) 133 return; 134 135 /* Enabling MSR intercept for x2APIC registers */ 136 svm_set_x2apic_msr_interception(svm, true); 137} 138 139/* Note: 140 * This function is called from IOMMU driver to notify 141 * SVM to schedule in a particular vCPU of a particular VM. 142 */ 143int avic_ga_log_notifier(u32 ga_tag) 144{ 145 unsigned long flags; 146 struct kvm_svm *kvm_svm; 147 struct kvm_vcpu *vcpu = NULL; 148 u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag); 149 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); 150 151 pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); 152 trace_kvm_avic_ga_log(vm_id, vcpu_id); 153 154 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 155 hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { 156 if (kvm_svm->avic_vm_id != vm_id) 157 continue; 158 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id); 159 break; 160 } 161 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 162 163 /* Note: 164 * At this point, the IOMMU should have already set the pending 165 * bit in the vAPIC backing page. So, we just need to schedule 166 * in the vcpu. 167 */ 168 if (vcpu) 169 kvm_vcpu_wake_up(vcpu); 170 171 return 0; 172} 173 174void avic_vm_destroy(struct kvm *kvm) 175{ 176 unsigned long flags; 177 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 178 179 if (!enable_apicv) 180 return; 181 182 if (kvm_svm->avic_logical_id_table_page) 183 __free_page(kvm_svm->avic_logical_id_table_page); 184 if (kvm_svm->avic_physical_id_table_page) 185 __free_page(kvm_svm->avic_physical_id_table_page); 186 187 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 188 hash_del(&kvm_svm->hnode); 189 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 190} 191 192int avic_vm_init(struct kvm *kvm) 193{ 194 unsigned long flags; 195 int err = -ENOMEM; 196 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 197 struct kvm_svm *k2; 198 struct page *p_page; 199 struct page *l_page; 200 u32 vm_id; 201 202 if (!enable_apicv) 203 return 0; 204 205 /* Allocating physical APIC ID table (4KB) */ 206 p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 207 if (!p_page) 208 goto free_avic; 209 210 kvm_svm->avic_physical_id_table_page = p_page; 211 212 /* Allocating logical APIC ID table (4KB) */ 213 l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 214 if (!l_page) 215 goto free_avic; 216 217 kvm_svm->avic_logical_id_table_page = l_page; 218 219 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 220 again: 221 vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; 222 if (vm_id == 0) { /* id is 1-based, zero is not okay */ 223 next_vm_id_wrapped = 1; 224 goto again; 225 } 226 /* Is it still in use? Only possible if wrapped at least once */ 227 if (next_vm_id_wrapped) { 228 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) { 229 if (k2->avic_vm_id == vm_id) 230 goto again; 231 } 232 } 233 kvm_svm->avic_vm_id = vm_id; 234 hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); 235 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 236 237 return 0; 238 239free_avic: 240 avic_vm_destroy(kvm); 241 return err; 242} 243 244void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb) 245{ 246 struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); 247 phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); 248 phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); 249 phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page)); 250 251 vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK; 252 vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK; 253 vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK; 254 vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE & VMCB_AVIC_APIC_BAR_MASK; 255 256 if (kvm_apicv_activated(svm->vcpu.kvm)) 257 avic_activate_vmcb(svm); 258 else 259 avic_deactivate_vmcb(svm); 260} 261 262static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, 263 unsigned int index) 264{ 265 u64 *avic_physical_id_table; 266 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 267 268 if ((!x2avic_enabled && index > AVIC_MAX_PHYSICAL_ID) || 269 (index > X2AVIC_MAX_PHYSICAL_ID)) 270 return NULL; 271 272 avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page); 273 274 return &avic_physical_id_table[index]; 275} 276 277static int avic_init_backing_page(struct kvm_vcpu *vcpu) 278{ 279 u64 *entry, new_entry; 280 int id = vcpu->vcpu_id; 281 struct vcpu_svm *svm = to_svm(vcpu); 282 283 if ((!x2avic_enabled && id > AVIC_MAX_PHYSICAL_ID) || 284 (id > X2AVIC_MAX_PHYSICAL_ID)) 285 return -EINVAL; 286 287 if (!vcpu->arch.apic->regs) 288 return -EINVAL; 289 290 if (kvm_apicv_activated(vcpu->kvm)) { 291 int ret; 292 293 /* 294 * Note, AVIC hardware walks the nested page table to check 295 * permissions, but does not use the SPA address specified in 296 * the leaf SPTE since it uses address in the AVIC_BACKING_PAGE 297 * pointer field of the VMCB. 298 */ 299 ret = kvm_alloc_apic_access_page(vcpu->kvm); 300 if (ret) 301 return ret; 302 } 303 304 svm->avic_backing_page = virt_to_page(vcpu->arch.apic->regs); 305 306 /* Setting AVIC backing page address in the phy APIC ID table */ 307 entry = avic_get_physical_id_entry(vcpu, id); 308 if (!entry) 309 return -EINVAL; 310 311 new_entry = __sme_set((page_to_phys(svm->avic_backing_page) & 312 AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) | 313 AVIC_PHYSICAL_ID_ENTRY_VALID_MASK); 314 WRITE_ONCE(*entry, new_entry); 315 316 svm->avic_physical_id_cache = entry; 317 318 return 0; 319} 320 321void avic_ring_doorbell(struct kvm_vcpu *vcpu) 322{ 323 /* 324 * Note, the vCPU could get migrated to a different pCPU at any point, 325 * which could result in signalling the wrong/previous pCPU. But if 326 * that happens the vCPU is guaranteed to do a VMRUN (after being 327 * migrated) and thus will process pending interrupts, i.e. a doorbell 328 * is not needed (and the spurious one is harmless). 329 */ 330 int cpu = READ_ONCE(vcpu->cpu); 331 332 if (cpu != get_cpu()) { 333 wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu)); 334 trace_kvm_avic_doorbell(vcpu->vcpu_id, kvm_cpu_get_apicid(cpu)); 335 } 336 put_cpu(); 337} 338 339 340static void avic_kick_vcpu(struct kvm_vcpu *vcpu, u32 icrl) 341{ 342 vcpu->arch.apic->irr_pending = true; 343 svm_complete_interrupt_delivery(vcpu, 344 icrl & APIC_MODE_MASK, 345 icrl & APIC_INT_LEVELTRIG, 346 icrl & APIC_VECTOR_MASK); 347} 348 349static void avic_kick_vcpu_by_physical_id(struct kvm *kvm, u32 physical_id, 350 u32 icrl) 351{ 352 /* 353 * KVM inhibits AVIC if any vCPU ID diverges from the vCPUs APIC ID, 354 * i.e. APIC ID == vCPU ID. 355 */ 356 struct kvm_vcpu *target_vcpu = kvm_get_vcpu_by_id(kvm, physical_id); 357 358 /* Once again, nothing to do if the target vCPU doesn't exist. */ 359 if (unlikely(!target_vcpu)) 360 return; 361 362 avic_kick_vcpu(target_vcpu, icrl); 363} 364 365static void avic_kick_vcpu_by_logical_id(struct kvm *kvm, u32 *avic_logical_id_table, 366 u32 logid_index, u32 icrl) 367{ 368 u32 physical_id; 369 370 if (avic_logical_id_table) { 371 u32 logid_entry = avic_logical_id_table[logid_index]; 372 373 /* Nothing to do if the logical destination is invalid. */ 374 if (unlikely(!(logid_entry & AVIC_LOGICAL_ID_ENTRY_VALID_MASK))) 375 return; 376 377 physical_id = logid_entry & 378 AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; 379 } else { 380 /* 381 * For x2APIC, the logical APIC ID is a read-only value that is 382 * derived from the x2APIC ID, thus the x2APIC ID can be found 383 * by reversing the calculation (stored in logid_index). Note, 384 * bits 31:20 of the x2APIC ID aren't propagated to the logical 385 * ID, but KVM limits the x2APIC ID limited to KVM_MAX_VCPU_IDS. 386 */ 387 physical_id = logid_index; 388 } 389 390 avic_kick_vcpu_by_physical_id(kvm, physical_id, icrl); 391} 392 393/* 394 * A fast-path version of avic_kick_target_vcpus(), which attempts to match 395 * destination APIC ID to vCPU without looping through all vCPUs. 396 */ 397static int avic_kick_target_vcpus_fast(struct kvm *kvm, struct kvm_lapic *source, 398 u32 icrl, u32 icrh, u32 index) 399{ 400 int dest_mode = icrl & APIC_DEST_MASK; 401 int shorthand = icrl & APIC_SHORT_MASK; 402 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 403 u32 dest; 404 405 if (shorthand != APIC_DEST_NOSHORT) 406 return -EINVAL; 407 408 if (apic_x2apic_mode(source)) 409 dest = icrh; 410 else 411 dest = GET_XAPIC_DEST_FIELD(icrh); 412 413 if (dest_mode == APIC_DEST_PHYSICAL) { 414 /* broadcast destination, use slow path */ 415 if (apic_x2apic_mode(source) && dest == X2APIC_BROADCAST) 416 return -EINVAL; 417 if (!apic_x2apic_mode(source) && dest == APIC_BROADCAST) 418 return -EINVAL; 419 420 if (WARN_ON_ONCE(dest != index)) 421 return -EINVAL; 422 423 avic_kick_vcpu_by_physical_id(kvm, dest, icrl); 424 } else { 425 u32 *avic_logical_id_table; 426 unsigned long bitmap, i; 427 u32 cluster; 428 429 if (apic_x2apic_mode(source)) { 430 /* 16 bit dest mask, 16 bit cluster id */ 431 bitmap = dest & 0xFFFF; 432 cluster = (dest >> 16) << 4; 433 } else if (kvm_lapic_get_reg(source, APIC_DFR) == APIC_DFR_FLAT) { 434 /* 8 bit dest mask*/ 435 bitmap = dest; 436 cluster = 0; 437 } else { 438 /* 4 bit desk mask, 4 bit cluster id */ 439 bitmap = dest & 0xF; 440 cluster = (dest >> 4) << 2; 441 } 442 443 /* Nothing to do if there are no destinations in the cluster. */ 444 if (unlikely(!bitmap)) 445 return 0; 446 447 if (apic_x2apic_mode(source)) 448 avic_logical_id_table = NULL; 449 else 450 avic_logical_id_table = page_address(kvm_svm->avic_logical_id_table_page); 451 452 /* 453 * AVIC is inhibited if vCPUs aren't mapped 1:1 with logical 454 * IDs, thus each bit in the destination is guaranteed to map 455 * to at most one vCPU. 456 */ 457 for_each_set_bit(i, &bitmap, 16) 458 avic_kick_vcpu_by_logical_id(kvm, avic_logical_id_table, 459 cluster + i, icrl); 460 } 461 462 return 0; 463} 464 465static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, 466 u32 icrl, u32 icrh, u32 index) 467{ 468 u32 dest = apic_x2apic_mode(source) ? icrh : GET_XAPIC_DEST_FIELD(icrh); 469 unsigned long i; 470 struct kvm_vcpu *vcpu; 471 472 if (!avic_kick_target_vcpus_fast(kvm, source, icrl, icrh, index)) 473 return; 474 475 trace_kvm_avic_kick_vcpu_slowpath(icrh, icrl, index); 476 477 /* 478 * Wake any target vCPUs that are blocking, i.e. waiting for a wake 479 * event. There's no need to signal doorbells, as hardware has handled 480 * vCPUs that were in guest at the time of the IPI, and vCPUs that have 481 * since entered the guest will have processed pending IRQs at VMRUN. 482 */ 483 kvm_for_each_vcpu(i, vcpu, kvm) { 484 if (kvm_apic_match_dest(vcpu, source, icrl & APIC_SHORT_MASK, 485 dest, icrl & APIC_DEST_MASK)) 486 avic_kick_vcpu(vcpu, icrl); 487 } 488} 489 490int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) 491{ 492 struct vcpu_svm *svm = to_svm(vcpu); 493 u32 icrh = svm->vmcb->control.exit_info_1 >> 32; 494 u32 icrl = svm->vmcb->control.exit_info_1; 495 u32 id = svm->vmcb->control.exit_info_2 >> 32; 496 u32 index = svm->vmcb->control.exit_info_2 & 0x1FF; 497 struct kvm_lapic *apic = vcpu->arch.apic; 498 499 trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index); 500 501 switch (id) { 502 case AVIC_IPI_FAILURE_INVALID_TARGET: 503 case AVIC_IPI_FAILURE_INVALID_INT_TYPE: 504 /* 505 * Emulate IPIs that are not handled by AVIC hardware, which 506 * only virtualizes Fixed, Edge-Triggered INTRs, and falls over 507 * if _any_ targets are invalid, e.g. if the logical mode mask 508 * is a superset of running vCPUs. 509 * 510 * The exit is a trap, e.g. ICR holds the correct value and RIP 511 * has been advanced, KVM is responsible only for emulating the 512 * IPI. Sadly, hardware may sometimes leave the BUSY flag set, 513 * in which case KVM needs to emulate the ICR write as well in 514 * order to clear the BUSY flag. 515 */ 516 if (icrl & APIC_ICR_BUSY) 517 kvm_apic_write_nodecode(vcpu, APIC_ICR); 518 else 519 kvm_apic_send_ipi(apic, icrl, icrh); 520 break; 521 case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: 522 /* 523 * At this point, we expect that the AVIC HW has already 524 * set the appropriate IRR bits on the valid target 525 * vcpus. So, we just need to kick the appropriate vcpu. 526 */ 527 avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh, index); 528 break; 529 case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: 530 WARN_ONCE(1, "Invalid backing page\n"); 531 break; 532 case AVIC_IPI_FAILURE_INVALID_IPI_VECTOR: 533 /* Invalid IPI with vector < 16 */ 534 break; 535 default: 536 vcpu_unimpl(vcpu, "Unknown avic incomplete IPI interception\n"); 537 } 538 539 return 1; 540} 541 542unsigned long avic_vcpu_get_apicv_inhibit_reasons(struct kvm_vcpu *vcpu) 543{ 544 if (is_guest_mode(vcpu)) 545 return APICV_INHIBIT_REASON_NESTED; 546 return 0; 547} 548 549static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) 550{ 551 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 552 u32 *logical_apic_id_table; 553 u32 cluster, index; 554 555 ldr = GET_APIC_LOGICAL_ID(ldr); 556 557 if (flat) { 558 cluster = 0; 559 } else { 560 cluster = (ldr >> 4); 561 if (cluster >= 0xf) 562 return NULL; 563 ldr &= 0xf; 564 } 565 if (!ldr || !is_power_of_2(ldr)) 566 return NULL; 567 568 index = __ffs(ldr); 569 if (WARN_ON_ONCE(index > 7)) 570 return NULL; 571 index += (cluster << 2); 572 573 logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page); 574 575 return &logical_apic_id_table[index]; 576} 577 578static void avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) 579{ 580 bool flat; 581 u32 *entry, new_entry; 582 583 flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT; 584 entry = avic_get_logical_id_entry(vcpu, ldr, flat); 585 if (!entry) 586 return; 587 588 new_entry = READ_ONCE(*entry); 589 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; 590 new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); 591 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; 592 WRITE_ONCE(*entry, new_entry); 593} 594 595static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) 596{ 597 struct vcpu_svm *svm = to_svm(vcpu); 598 bool flat = svm->dfr_reg == APIC_DFR_FLAT; 599 u32 *entry; 600 601 /* Note: x2AVIC does not use logical APIC ID table */ 602 if (apic_x2apic_mode(vcpu->arch.apic)) 603 return; 604 605 entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); 606 if (entry) 607 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry); 608} 609 610static void avic_handle_ldr_update(struct kvm_vcpu *vcpu) 611{ 612 struct vcpu_svm *svm = to_svm(vcpu); 613 u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); 614 u32 id = kvm_xapic_id(vcpu->arch.apic); 615 616 /* AVIC does not support LDR update for x2APIC */ 617 if (apic_x2apic_mode(vcpu->arch.apic)) 618 return; 619 620 if (ldr == svm->ldr_reg) 621 return; 622 623 avic_invalidate_logical_id_entry(vcpu); 624 625 svm->ldr_reg = ldr; 626 avic_ldr_write(vcpu, id, ldr); 627} 628 629static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) 630{ 631 struct vcpu_svm *svm = to_svm(vcpu); 632 u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); 633 634 if (svm->dfr_reg == dfr) 635 return; 636 637 avic_invalidate_logical_id_entry(vcpu); 638 svm->dfr_reg = dfr; 639} 640 641static int avic_unaccel_trap_write(struct kvm_vcpu *vcpu) 642{ 643 u32 offset = to_svm(vcpu)->vmcb->control.exit_info_1 & 644 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 645 646 switch (offset) { 647 case APIC_LDR: 648 avic_handle_ldr_update(vcpu); 649 break; 650 case APIC_DFR: 651 avic_handle_dfr_update(vcpu); 652 break; 653 case APIC_RRR: 654 /* Ignore writes to Read Remote Data, it's read-only. */ 655 return 1; 656 default: 657 break; 658 } 659 660 kvm_apic_write_nodecode(vcpu, offset); 661 return 1; 662} 663 664static bool is_avic_unaccelerated_access_trap(u32 offset) 665{ 666 bool ret = false; 667 668 switch (offset) { 669 case APIC_ID: 670 case APIC_EOI: 671 case APIC_RRR: 672 case APIC_LDR: 673 case APIC_DFR: 674 case APIC_SPIV: 675 case APIC_ESR: 676 case APIC_ICR: 677 case APIC_LVTT: 678 case APIC_LVTTHMR: 679 case APIC_LVTPC: 680 case APIC_LVT0: 681 case APIC_LVT1: 682 case APIC_LVTERR: 683 case APIC_TMICT: 684 case APIC_TDCR: 685 ret = true; 686 break; 687 default: 688 break; 689 } 690 return ret; 691} 692 693int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu) 694{ 695 struct vcpu_svm *svm = to_svm(vcpu); 696 int ret = 0; 697 u32 offset = svm->vmcb->control.exit_info_1 & 698 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 699 u32 vector = svm->vmcb->control.exit_info_2 & 700 AVIC_UNACCEL_ACCESS_VECTOR_MASK; 701 bool write = (svm->vmcb->control.exit_info_1 >> 32) & 702 AVIC_UNACCEL_ACCESS_WRITE_MASK; 703 bool trap = is_avic_unaccelerated_access_trap(offset); 704 705 trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset, 706 trap, write, vector); 707 if (trap) { 708 /* Handling Trap */ 709 WARN_ONCE(!write, "svm: Handling trap read.\n"); 710 ret = avic_unaccel_trap_write(vcpu); 711 } else { 712 /* Handling Fault */ 713 ret = kvm_emulate_instruction(vcpu, 0); 714 } 715 716 return ret; 717} 718 719int avic_init_vcpu(struct vcpu_svm *svm) 720{ 721 int ret; 722 struct kvm_vcpu *vcpu = &svm->vcpu; 723 724 if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm)) 725 return 0; 726 727 ret = avic_init_backing_page(vcpu); 728 if (ret) 729 return ret; 730 731 INIT_LIST_HEAD(&svm->ir_list); 732 spin_lock_init(&svm->ir_list_lock); 733 svm->dfr_reg = APIC_DFR_FLAT; 734 735 return ret; 736} 737 738void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu) 739{ 740 avic_handle_dfr_update(vcpu); 741 avic_handle_ldr_update(vcpu); 742} 743 744static int avic_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate) 745{ 746 int ret = 0; 747 unsigned long flags; 748 struct amd_svm_iommu_ir *ir; 749 struct vcpu_svm *svm = to_svm(vcpu); 750 751 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 752 return 0; 753 754 /* 755 * Here, we go through the per-vcpu ir_list to update all existing 756 * interrupt remapping table entry targeting this vcpu. 757 */ 758 spin_lock_irqsave(&svm->ir_list_lock, flags); 759 760 if (list_empty(&svm->ir_list)) 761 goto out; 762 763 list_for_each_entry(ir, &svm->ir_list, node) { 764 if (activate) 765 ret = amd_iommu_activate_guest_mode(ir->data); 766 else 767 ret = amd_iommu_deactivate_guest_mode(ir->data); 768 if (ret) 769 break; 770 } 771out: 772 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 773 return ret; 774} 775 776static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 777{ 778 unsigned long flags; 779 struct amd_svm_iommu_ir *cur; 780 781 spin_lock_irqsave(&svm->ir_list_lock, flags); 782 list_for_each_entry(cur, &svm->ir_list, node) { 783 if (cur->data != pi->ir_data) 784 continue; 785 list_del(&cur->node); 786 kfree(cur); 787 break; 788 } 789 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 790} 791 792static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 793{ 794 int ret = 0; 795 unsigned long flags; 796 struct amd_svm_iommu_ir *ir; 797 u64 entry; 798 799 /** 800 * In some cases, the existing irte is updated and re-set, 801 * so we need to check here if it's already been * added 802 * to the ir_list. 803 */ 804 if (pi->ir_data && (pi->prev_ga_tag != 0)) { 805 struct kvm *kvm = svm->vcpu.kvm; 806 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag); 807 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 808 struct vcpu_svm *prev_svm; 809 810 if (!prev_vcpu) { 811 ret = -EINVAL; 812 goto out; 813 } 814 815 prev_svm = to_svm(prev_vcpu); 816 svm_ir_list_del(prev_svm, pi); 817 } 818 819 /** 820 * Allocating new amd_iommu_pi_data, which will get 821 * add to the per-vcpu ir_list. 822 */ 823 ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); 824 if (!ir) { 825 ret = -ENOMEM; 826 goto out; 827 } 828 ir->data = pi->ir_data; 829 830 spin_lock_irqsave(&svm->ir_list_lock, flags); 831 832 /* 833 * Update the target pCPU for IOMMU doorbells if the vCPU is running. 834 * If the vCPU is NOT running, i.e. is blocking or scheduled out, KVM 835 * will update the pCPU info when the vCPU awkened and/or scheduled in. 836 * See also avic_vcpu_load(). 837 */ 838 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 839 if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) 840 amd_iommu_update_ga(entry & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK, 841 true, pi->ir_data); 842 843 list_add(&ir->node, &svm->ir_list); 844 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 845out: 846 return ret; 847} 848 849/* 850 * Note: 851 * The HW cannot support posting multicast/broadcast 852 * interrupts to a vCPU. So, we still use legacy interrupt 853 * remapping for these kind of interrupts. 854 * 855 * For lowest-priority interrupts, we only support 856 * those with single CPU as the destination, e.g. user 857 * configures the interrupts via /proc/irq or uses 858 * irqbalance to make the interrupts single-CPU. 859 */ 860static int 861get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, 862 struct vcpu_data *vcpu_info, struct vcpu_svm **svm) 863{ 864 struct kvm_lapic_irq irq; 865 struct kvm_vcpu *vcpu = NULL; 866 867 kvm_set_msi_irq(kvm, e, &irq); 868 869 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || 870 !kvm_irq_is_postable(&irq)) { 871 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n", 872 __func__, irq.vector); 873 return -1; 874 } 875 876 pr_debug("SVM: %s: use GA mode for irq %u\n", __func__, 877 irq.vector); 878 *svm = to_svm(vcpu); 879 vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page)); 880 vcpu_info->vector = irq.vector; 881 882 return 0; 883} 884 885/* 886 * avic_pi_update_irte - set IRTE for Posted-Interrupts 887 * 888 * @kvm: kvm 889 * @host_irq: host irq of the interrupt 890 * @guest_irq: gsi of the interrupt 891 * @set: set or unset PI 892 * returns 0 on success, < 0 on failure 893 */ 894int avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq, 895 uint32_t guest_irq, bool set) 896{ 897 struct kvm_kernel_irq_routing_entry *e; 898 struct kvm_irq_routing_table *irq_rt; 899 int idx, ret = 0; 900 901 if (!kvm_arch_has_assigned_device(kvm) || 902 !irq_remapping_cap(IRQ_POSTING_CAP)) 903 return 0; 904 905 pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n", 906 __func__, host_irq, guest_irq, set); 907 908 idx = srcu_read_lock(&kvm->irq_srcu); 909 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 910 911 if (guest_irq >= irq_rt->nr_rt_entries || 912 hlist_empty(&irq_rt->map[guest_irq])) { 913 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n", 914 guest_irq, irq_rt->nr_rt_entries); 915 goto out; 916 } 917 918 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { 919 struct vcpu_data vcpu_info; 920 struct vcpu_svm *svm = NULL; 921 922 if (e->type != KVM_IRQ_ROUTING_MSI) 923 continue; 924 925 /** 926 * Here, we setup with legacy mode in the following cases: 927 * 1. When cannot target interrupt to a specific vcpu. 928 * 2. Unsetting posted interrupt. 929 * 3. APIC virtualization is disabled for the vcpu. 930 * 4. IRQ has incompatible delivery mode (SMI, INIT, etc) 931 */ 932 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set && 933 kvm_vcpu_apicv_active(&svm->vcpu)) { 934 struct amd_iommu_pi_data pi; 935 936 /* Try to enable guest_mode in IRTE */ 937 pi.base = __sme_set(page_to_phys(svm->avic_backing_page) & 938 AVIC_HPA_MASK); 939 pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id, 940 svm->vcpu.vcpu_id); 941 pi.is_guest_mode = true; 942 pi.vcpu_data = &vcpu_info; 943 ret = irq_set_vcpu_affinity(host_irq, &pi); 944 945 /** 946 * Here, we successfully setting up vcpu affinity in 947 * IOMMU guest mode. Now, we need to store the posted 948 * interrupt information in a per-vcpu ir_list so that 949 * we can reference to them directly when we update vcpu 950 * scheduling information in IOMMU irte. 951 */ 952 if (!ret && pi.is_guest_mode) 953 svm_ir_list_add(svm, &pi); 954 } else { 955 /* Use legacy mode in IRTE */ 956 struct amd_iommu_pi_data pi; 957 958 /** 959 * Here, pi is used to: 960 * - Tell IOMMU to use legacy mode for this interrupt. 961 * - Retrieve ga_tag of prior interrupt remapping data. 962 */ 963 pi.prev_ga_tag = 0; 964 pi.is_guest_mode = false; 965 ret = irq_set_vcpu_affinity(host_irq, &pi); 966 967 /** 968 * Check if the posted interrupt was previously 969 * setup with the guest_mode by checking if the ga_tag 970 * was cached. If so, we need to clean up the per-vcpu 971 * ir_list. 972 */ 973 if (!ret && pi.prev_ga_tag) { 974 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag); 975 struct kvm_vcpu *vcpu; 976 977 vcpu = kvm_get_vcpu_by_id(kvm, id); 978 if (vcpu) 979 svm_ir_list_del(to_svm(vcpu), &pi); 980 } 981 } 982 983 if (!ret && svm) { 984 trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id, 985 e->gsi, vcpu_info.vector, 986 vcpu_info.pi_desc_addr, set); 987 } 988 989 if (ret < 0) { 990 pr_err("%s: failed to update PI IRTE\n", __func__); 991 goto out; 992 } 993 } 994 995 ret = 0; 996out: 997 srcu_read_unlock(&kvm->irq_srcu, idx); 998 return ret; 999} 1000 1001static inline int 1002avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r) 1003{ 1004 int ret = 0; 1005 struct amd_svm_iommu_ir *ir; 1006 struct vcpu_svm *svm = to_svm(vcpu); 1007 1008 lockdep_assert_held(&svm->ir_list_lock); 1009 1010 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 1011 return 0; 1012 1013 /* 1014 * Here, we go through the per-vcpu ir_list to update all existing 1015 * interrupt remapping table entry targeting this vcpu. 1016 */ 1017 if (list_empty(&svm->ir_list)) 1018 return 0; 1019 1020 list_for_each_entry(ir, &svm->ir_list, node) { 1021 ret = amd_iommu_update_ga(cpu, r, ir->data); 1022 if (ret) 1023 return ret; 1024 } 1025 return 0; 1026} 1027 1028void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1029{ 1030 u64 entry; 1031 int h_physical_id = kvm_cpu_get_apicid(cpu); 1032 struct vcpu_svm *svm = to_svm(vcpu); 1033 unsigned long flags; 1034 1035 lockdep_assert_preemption_disabled(); 1036 1037 if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) 1038 return; 1039 1040 /* 1041 * No need to update anything if the vCPU is blocking, i.e. if the vCPU 1042 * is being scheduled in after being preempted. The CPU entries in the 1043 * Physical APIC table and IRTE are consumed iff IsRun{ning} is '1'. 1044 * If the vCPU was migrated, its new CPU value will be stuffed when the 1045 * vCPU unblocks. 1046 */ 1047 if (kvm_vcpu_is_blocking(vcpu)) 1048 return; 1049 1050 /* 1051 * Grab the per-vCPU interrupt remapping lock even if the VM doesn't 1052 * _currently_ have assigned devices, as that can change. Holding 1053 * ir_list_lock ensures that either svm_ir_list_add() will consume 1054 * up-to-date entry information, or that this task will wait until 1055 * svm_ir_list_add() completes to set the new target pCPU. 1056 */ 1057 spin_lock_irqsave(&svm->ir_list_lock, flags); 1058 1059 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 1060 WARN_ON_ONCE(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); 1061 1062 entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; 1063 entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); 1064 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 1065 1066 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 1067 avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, true); 1068 1069 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 1070} 1071 1072void avic_vcpu_put(struct kvm_vcpu *vcpu) 1073{ 1074 u64 entry; 1075 struct vcpu_svm *svm = to_svm(vcpu); 1076 unsigned long flags; 1077 1078 lockdep_assert_preemption_disabled(); 1079 1080 /* 1081 * Note, reading the Physical ID entry outside of ir_list_lock is safe 1082 * as only the pCPU that has loaded (or is loading) the vCPU is allowed 1083 * to modify the entry, and preemption is disabled. I.e. the vCPU 1084 * can't be scheduled out and thus avic_vcpu_{put,load}() can't run 1085 * recursively. 1086 */ 1087 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 1088 1089 /* Nothing to do if IsRunning == '0' due to vCPU blocking. */ 1090 if (!(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)) 1091 return; 1092 1093 /* 1094 * Take and hold the per-vCPU interrupt remapping lock while updating 1095 * the Physical ID entry even though the lock doesn't protect against 1096 * multiple writers (see above). Holding ir_list_lock ensures that 1097 * either svm_ir_list_add() will consume up-to-date entry information, 1098 * or that this task will wait until svm_ir_list_add() completes to 1099 * mark the vCPU as not running. 1100 */ 1101 spin_lock_irqsave(&svm->ir_list_lock, flags); 1102 1103 avic_update_iommu_vcpu_affinity(vcpu, -1, 0); 1104 1105 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 1106 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 1107 1108 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 1109 1110} 1111 1112void avic_refresh_virtual_apic_mode(struct kvm_vcpu *vcpu) 1113{ 1114 struct vcpu_svm *svm = to_svm(vcpu); 1115 struct vmcb *vmcb = svm->vmcb01.ptr; 1116 1117 if (!lapic_in_kernel(vcpu) || !enable_apicv) 1118 return; 1119 1120 if (kvm_vcpu_apicv_active(vcpu)) { 1121 /** 1122 * During AVIC temporary deactivation, guest could update 1123 * APIC ID, DFR and LDR registers, which would not be trapped 1124 * by avic_unaccelerated_access_interception(). In this case, 1125 * we need to check and update the AVIC logical APIC ID table 1126 * accordingly before re-activating. 1127 */ 1128 avic_apicv_post_state_restore(vcpu); 1129 avic_activate_vmcb(svm); 1130 } else { 1131 avic_deactivate_vmcb(svm); 1132 } 1133 vmcb_mark_dirty(vmcb, VMCB_AVIC); 1134} 1135 1136void avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 1137{ 1138 bool activated = kvm_vcpu_apicv_active(vcpu); 1139 1140 if (!enable_apicv) 1141 return; 1142 1143 avic_refresh_virtual_apic_mode(vcpu); 1144 1145 if (activated) 1146 avic_vcpu_load(vcpu, vcpu->cpu); 1147 else 1148 avic_vcpu_put(vcpu); 1149 1150 avic_set_pi_irte_mode(vcpu, activated); 1151} 1152 1153void avic_vcpu_blocking(struct kvm_vcpu *vcpu) 1154{ 1155 if (!kvm_vcpu_apicv_active(vcpu)) 1156 return; 1157 1158 /* 1159 * Unload the AVIC when the vCPU is about to block, _before_ 1160 * the vCPU actually blocks. 1161 * 1162 * Any IRQs that arrive before IsRunning=0 will not cause an 1163 * incomplete IPI vmexit on the source, therefore vIRR will also 1164 * be checked by kvm_vcpu_check_block() before blocking. The 1165 * memory barrier implicit in set_current_state orders writing 1166 * IsRunning=0 before reading the vIRR. The processor needs a 1167 * matching memory barrier on interrupt delivery between writing 1168 * IRR and reading IsRunning; the lack of this barrier might be 1169 * the cause of errata #1235). 1170 */ 1171 avic_vcpu_put(vcpu); 1172} 1173 1174void avic_vcpu_unblocking(struct kvm_vcpu *vcpu) 1175{ 1176 if (!kvm_vcpu_apicv_active(vcpu)) 1177 return; 1178 1179 avic_vcpu_load(vcpu, vcpu->cpu); 1180} 1181 1182/* 1183 * Note: 1184 * - The module param avic enable both xAPIC and x2APIC mode. 1185 * - Hypervisor can support both xAVIC and x2AVIC in the same guest. 1186 * - The mode can be switched at run-time. 1187 */ 1188bool avic_hardware_setup(void) 1189{ 1190 if (!npt_enabled) 1191 return false; 1192 1193 /* AVIC is a prerequisite for x2AVIC. */ 1194 if (!boot_cpu_has(X86_FEATURE_AVIC) && !force_avic) { 1195 if (boot_cpu_has(X86_FEATURE_X2AVIC)) { 1196 pr_warn(FW_BUG "Cannot support x2AVIC due to AVIC is disabled"); 1197 pr_warn(FW_BUG "Try enable AVIC using force_avic option"); 1198 } 1199 return false; 1200 } 1201 1202 if (boot_cpu_has(X86_FEATURE_AVIC)) { 1203 pr_info("AVIC enabled\n"); 1204 } else if (force_avic) { 1205 /* 1206 * Some older systems does not advertise AVIC support. 1207 * See Revision Guide for specific AMD processor for more detail. 1208 */ 1209 pr_warn("AVIC is not supported in CPUID but force enabled"); 1210 pr_warn("Your system might crash and burn"); 1211 } 1212 1213 /* AVIC is a prerequisite for x2AVIC. */ 1214 x2avic_enabled = boot_cpu_has(X86_FEATURE_X2AVIC); 1215 if (x2avic_enabled) 1216 pr_info("x2AVIC enabled\n"); 1217 1218 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); 1219 1220 return true; 1221} 1222