1// SPDX-License-Identifier: GPL-2.0 2/* 3 * AMD Encrypted Register State Support 4 * 5 * Author: Joerg Roedel <jroedel@suse.de> 6 */ 7 8/* 9 * misc.h needs to be first because it knows how to include the other kernel 10 * headers in the pre-decompression code in a way that does not break 11 * compilation. 12 */ 13#include "misc.h" 14 15#include <asm/bootparam.h> 16#include <asm/pgtable_types.h> 17#include <asm/sev.h> 18#include <asm/trapnr.h> 19#include <asm/trap_pf.h> 20#include <asm/msr-index.h> 21#include <asm/fpu/xcr.h> 22#include <asm/ptrace.h> 23#include <asm/svm.h> 24#include <asm/cpuid.h> 25 26#include "error.h" 27#include "../msr.h" 28 29static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE); 30struct ghcb *boot_ghcb; 31 32/* 33 * Copy a version of this function here - insn-eval.c can't be used in 34 * pre-decompression code. 35 */ 36static bool insn_has_rep_prefix(struct insn *insn) 37{ 38 insn_byte_t p; 39 int i; 40 41 insn_get_prefixes(insn); 42 43 for_each_insn_prefix(insn, i, p) { 44 if (p == 0xf2 || p == 0xf3) 45 return true; 46 } 47 48 return false; 49} 50 51/* 52 * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and 53 * doesn't use segments. 54 */ 55static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx) 56{ 57 return 0UL; 58} 59 60static inline u64 sev_es_rd_ghcb_msr(void) 61{ 62 struct msr m; 63 64 boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m); 65 66 return m.q; 67} 68 69static inline void sev_es_wr_ghcb_msr(u64 val) 70{ 71 struct msr m; 72 73 m.q = val; 74 boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m); 75} 76 77static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) 78{ 79 char buffer[MAX_INSN_SIZE]; 80 int ret; 81 82 memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); 83 84 ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64); 85 if (ret < 0) 86 return ES_DECODE_FAILED; 87 88 return ES_OK; 89} 90 91static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, 92 void *dst, char *buf, size_t size) 93{ 94 memcpy(dst, buf, size); 95 96 return ES_OK; 97} 98 99static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, 100 void *src, char *buf, size_t size) 101{ 102 memcpy(buf, src, size); 103 104 return ES_OK; 105} 106 107static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size) 108{ 109 return ES_OK; 110} 111 112static bool fault_in_kernel_space(unsigned long address) 113{ 114 return false; 115} 116 117#undef __init 118#define __init 119 120#undef __head 121#define __head 122 123#define __BOOT_COMPRESSED 124 125/* Basic instruction decoding support needed */ 126#include "../../lib/inat.c" 127#include "../../lib/insn.c" 128 129/* Include code for early handlers */ 130#include "../../kernel/sev-shared.c" 131 132bool sev_snp_enabled(void) 133{ 134 return sev_status & MSR_AMD64_SEV_SNP_ENABLED; 135} 136 137static void __page_state_change(unsigned long paddr, enum psc_op op) 138{ 139 u64 val; 140 141 if (!sev_snp_enabled()) 142 return; 143 144 /* 145 * If private -> shared then invalidate the page before requesting the 146 * state change in the RMP table. 147 */ 148 if (op == SNP_PAGE_STATE_SHARED && pvalidate(paddr, RMP_PG_SIZE_4K, 0)) 149 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE); 150 151 /* Issue VMGEXIT to change the page state in RMP table. */ 152 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op)); 153 VMGEXIT(); 154 155 /* Read the response of the VMGEXIT. */ 156 val = sev_es_rd_ghcb_msr(); 157 if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val)) 158 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 159 160 /* 161 * Now that page state is changed in the RMP table, validate it so that it is 162 * consistent with the RMP entry. 163 */ 164 if (op == SNP_PAGE_STATE_PRIVATE && pvalidate(paddr, RMP_PG_SIZE_4K, 1)) 165 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE); 166} 167 168void snp_set_page_private(unsigned long paddr) 169{ 170 __page_state_change(paddr, SNP_PAGE_STATE_PRIVATE); 171} 172 173void snp_set_page_shared(unsigned long paddr) 174{ 175 __page_state_change(paddr, SNP_PAGE_STATE_SHARED); 176} 177 178static bool early_setup_ghcb(void) 179{ 180 if (set_page_decrypted((unsigned long)&boot_ghcb_page)) 181 return false; 182 183 /* Page is now mapped decrypted, clear it */ 184 memset(&boot_ghcb_page, 0, sizeof(boot_ghcb_page)); 185 186 boot_ghcb = &boot_ghcb_page; 187 188 /* Initialize lookup tables for the instruction decoder */ 189 inat_init_tables(); 190 191 /* SNP guest requires the GHCB GPA must be registered */ 192 if (sev_snp_enabled()) 193 snp_register_ghcb_early(__pa(&boot_ghcb_page)); 194 195 return true; 196} 197 198static phys_addr_t __snp_accept_memory(struct snp_psc_desc *desc, 199 phys_addr_t pa, phys_addr_t pa_end) 200{ 201 struct psc_hdr *hdr; 202 struct psc_entry *e; 203 unsigned int i; 204 205 hdr = &desc->hdr; 206 memset(hdr, 0, sizeof(*hdr)); 207 208 e = desc->entries; 209 210 i = 0; 211 while (pa < pa_end && i < VMGEXIT_PSC_MAX_ENTRY) { 212 hdr->end_entry = i; 213 214 e->gfn = pa >> PAGE_SHIFT; 215 e->operation = SNP_PAGE_STATE_PRIVATE; 216 if (IS_ALIGNED(pa, PMD_SIZE) && (pa_end - pa) >= PMD_SIZE) { 217 e->pagesize = RMP_PG_SIZE_2M; 218 pa += PMD_SIZE; 219 } else { 220 e->pagesize = RMP_PG_SIZE_4K; 221 pa += PAGE_SIZE; 222 } 223 224 e++; 225 i++; 226 } 227 228 if (vmgexit_psc(boot_ghcb, desc)) 229 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 230 231 pvalidate_pages(desc); 232 233 return pa; 234} 235 236void snp_accept_memory(phys_addr_t start, phys_addr_t end) 237{ 238 struct snp_psc_desc desc = {}; 239 unsigned int i; 240 phys_addr_t pa; 241 242 if (!boot_ghcb && !early_setup_ghcb()) 243 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 244 245 pa = start; 246 while (pa < end) 247 pa = __snp_accept_memory(&desc, pa, end); 248} 249 250void sev_es_shutdown_ghcb(void) 251{ 252 if (!boot_ghcb) 253 return; 254 255 if (!sev_es_check_cpu_features()) 256 error("SEV-ES CPU Features missing."); 257 258 /* 259 * GHCB Page must be flushed from the cache and mapped encrypted again. 260 * Otherwise the running kernel will see strange cache effects when 261 * trying to use that page. 262 */ 263 if (set_page_encrypted((unsigned long)&boot_ghcb_page)) 264 error("Can't map GHCB page encrypted"); 265 266 /* 267 * GHCB page is mapped encrypted again and flushed from the cache. 268 * Mark it non-present now to catch bugs when #VC exceptions trigger 269 * after this point. 270 */ 271 if (set_page_non_present((unsigned long)&boot_ghcb_page)) 272 error("Can't unmap GHCB page"); 273} 274 275static void __noreturn sev_es_ghcb_terminate(struct ghcb *ghcb, unsigned int set, 276 unsigned int reason, u64 exit_info_2) 277{ 278 u64 exit_info_1 = SVM_VMGEXIT_TERM_REASON(set, reason); 279 280 vc_ghcb_invalidate(ghcb); 281 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_TERM_REQUEST); 282 ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 283 ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 284 285 sev_es_wr_ghcb_msr(__pa(ghcb)); 286 VMGEXIT(); 287 288 while (true) 289 asm volatile("hlt\n" : : : "memory"); 290} 291 292bool sev_es_check_ghcb_fault(unsigned long address) 293{ 294 /* Check whether the fault was on the GHCB page */ 295 return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page); 296} 297 298void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) 299{ 300 struct es_em_ctxt ctxt; 301 enum es_result result; 302 303 if (!boot_ghcb && !early_setup_ghcb()) 304 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 305 306 vc_ghcb_invalidate(boot_ghcb); 307 result = vc_init_em_ctxt(&ctxt, regs, exit_code); 308 if (result != ES_OK) 309 goto finish; 310 311 result = vc_check_opcode_bytes(&ctxt, exit_code); 312 if (result != ES_OK) 313 goto finish; 314 315 switch (exit_code) { 316 case SVM_EXIT_RDTSC: 317 case SVM_EXIT_RDTSCP: 318 result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code); 319 break; 320 case SVM_EXIT_IOIO: 321 result = vc_handle_ioio(boot_ghcb, &ctxt); 322 break; 323 case SVM_EXIT_CPUID: 324 result = vc_handle_cpuid(boot_ghcb, &ctxt); 325 break; 326 default: 327 result = ES_UNSUPPORTED; 328 break; 329 } 330 331finish: 332 if (result == ES_OK) 333 vc_finish_insn(&ctxt); 334 else if (result != ES_RETRY) 335 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 336} 337 338static void enforce_vmpl0(void) 339{ 340 u64 attrs; 341 int err; 342 343 /* 344 * RMPADJUST modifies RMP permissions of a lesser-privileged (numerically 345 * higher) privilege level. Here, clear the VMPL1 permission mask of the 346 * GHCB page. If the guest is not running at VMPL0, this will fail. 347 * 348 * If the guest is running at VMPL0, it will succeed. Even if that operation 349 * modifies permission bits, it is still ok to do so currently because Linux 350 * SNP guests are supported only on VMPL0 so VMPL1 or higher permission masks 351 * changing is a don't-care. 352 */ 353 attrs = 1; 354 if (rmpadjust((unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, attrs)) 355 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0); 356} 357 358/* 359 * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need 360 * guest side implementation for proper functioning of the guest. If any 361 * of these features are enabled in the hypervisor but are lacking guest 362 * side implementation, the behavior of the guest will be undefined. The 363 * guest could fail in non-obvious way making it difficult to debug. 364 * 365 * As the behavior of reserved feature bits is unknown to be on the 366 * safe side add them to the required features mask. 367 */ 368#define SNP_FEATURES_IMPL_REQ (MSR_AMD64_SNP_VTOM | \ 369 MSR_AMD64_SNP_REFLECT_VC | \ 370 MSR_AMD64_SNP_RESTRICTED_INJ | \ 371 MSR_AMD64_SNP_ALT_INJ | \ 372 MSR_AMD64_SNP_DEBUG_SWAP | \ 373 MSR_AMD64_SNP_VMPL_SSS | \ 374 MSR_AMD64_SNP_SECURE_TSC | \ 375 MSR_AMD64_SNP_VMGEXIT_PARAM | \ 376 MSR_AMD64_SNP_VMSA_REG_PROT | \ 377 MSR_AMD64_SNP_RESERVED_BIT13 | \ 378 MSR_AMD64_SNP_RESERVED_BIT15 | \ 379 MSR_AMD64_SNP_RESERVED_MASK) 380 381/* 382 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented 383 * by the guest kernel. As and when a new feature is implemented in the 384 * guest kernel, a corresponding bit should be added to the mask. 385 */ 386#define SNP_FEATURES_PRESENT MSR_AMD64_SNP_DEBUG_SWAP 387 388u64 snp_get_unsupported_features(u64 status) 389{ 390 if (!(status & MSR_AMD64_SEV_SNP_ENABLED)) 391 return 0; 392 393 return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT; 394} 395 396void snp_check_features(void) 397{ 398 u64 unsupported; 399 400 /* 401 * Terminate the boot if hypervisor has enabled any feature lacking 402 * guest side implementation. Pass on the unsupported features mask through 403 * EXIT_INFO_2 of the GHCB protocol so that those features can be reported 404 * as part of the guest boot failure. 405 */ 406 unsupported = snp_get_unsupported_features(sev_status); 407 if (unsupported) { 408 if (ghcb_version < 2 || (!boot_ghcb && !early_setup_ghcb())) 409 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 410 411 sev_es_ghcb_terminate(boot_ghcb, SEV_TERM_SET_GEN, 412 GHCB_SNP_UNSUPPORTED, unsupported); 413 } 414} 415 416/* 417 * sev_check_cpu_support - Check for SEV support in the CPU capabilities 418 * 419 * Returns < 0 if SEV is not supported, otherwise the position of the 420 * encryption bit in the page table descriptors. 421 */ 422static int sev_check_cpu_support(void) 423{ 424 unsigned int eax, ebx, ecx, edx; 425 426 /* Check for the SME/SEV support leaf */ 427 eax = 0x80000000; 428 ecx = 0; 429 native_cpuid(&eax, &ebx, &ecx, &edx); 430 if (eax < 0x8000001f) 431 return -ENODEV; 432 433 /* 434 * Check for the SME/SEV feature: 435 * CPUID Fn8000_001F[EAX] 436 * - Bit 0 - Secure Memory Encryption support 437 * - Bit 1 - Secure Encrypted Virtualization support 438 * CPUID Fn8000_001F[EBX] 439 * - Bits 5:0 - Pagetable bit position used to indicate encryption 440 */ 441 eax = 0x8000001f; 442 ecx = 0; 443 native_cpuid(&eax, &ebx, &ecx, &edx); 444 /* Check whether SEV is supported */ 445 if (!(eax & BIT(1))) 446 return -ENODEV; 447 448 return ebx & 0x3f; 449} 450 451void sev_enable(struct boot_params *bp) 452{ 453 struct msr m; 454 int bitpos; 455 bool snp; 456 457 /* 458 * bp->cc_blob_address should only be set by boot/compressed kernel. 459 * Initialize it to 0 to ensure that uninitialized values from 460 * buggy bootloaders aren't propagated. 461 */ 462 if (bp) 463 bp->cc_blob_address = 0; 464 465 /* 466 * Do an initial SEV capability check before snp_init() which 467 * loads the CPUID page and the same checks afterwards are done 468 * without the hypervisor and are trustworthy. 469 * 470 * If the HV fakes SEV support, the guest will crash'n'burn 471 * which is good enough. 472 */ 473 474 if (sev_check_cpu_support() < 0) 475 return; 476 477 /* 478 * Setup/preliminary detection of SNP. This will be sanity-checked 479 * against CPUID/MSR values later. 480 */ 481 snp = snp_init(bp); 482 483 /* Now repeat the checks with the SNP CPUID table. */ 484 485 bitpos = sev_check_cpu_support(); 486 if (bitpos < 0) { 487 if (snp) 488 error("SEV-SNP support indicated by CC blob, but not CPUID."); 489 return; 490 } 491 492 /* Set the SME mask if this is an SEV guest. */ 493 boot_rdmsr(MSR_AMD64_SEV, &m); 494 sev_status = m.q; 495 if (!(sev_status & MSR_AMD64_SEV_ENABLED)) 496 return; 497 498 /* Negotiate the GHCB protocol version. */ 499 if (sev_status & MSR_AMD64_SEV_ES_ENABLED) { 500 if (!sev_es_negotiate_protocol()) 501 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED); 502 } 503 504 /* 505 * SNP is supported in v2 of the GHCB spec which mandates support for HV 506 * features. 507 */ 508 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) { 509 if (!(get_hv_features() & GHCB_HV_FT_SNP)) 510 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 511 512 enforce_vmpl0(); 513 } 514 515 if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED)) 516 error("SEV-SNP supported indicated by CC blob, but not SEV status MSR."); 517 518 sme_me_mask = BIT_ULL(bitpos); 519} 520 521/* 522 * sev_get_status - Retrieve the SEV status mask 523 * 524 * Returns 0 if the CPU is not SEV capable, otherwise the value of the 525 * AMD64_SEV MSR. 526 */ 527u64 sev_get_status(void) 528{ 529 struct msr m; 530 531 if (sev_check_cpu_support() < 0) 532 return 0; 533 534 boot_rdmsr(MSR_AMD64_SEV, &m); 535 return m.q; 536} 537 538/* Search for Confidential Computing blob in the EFI config table. */ 539static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp) 540{ 541 unsigned long cfg_table_pa; 542 unsigned int cfg_table_len; 543 int ret; 544 545 ret = efi_get_conf_table(bp, &cfg_table_pa, &cfg_table_len); 546 if (ret) 547 return NULL; 548 549 return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_table_pa, 550 cfg_table_len, 551 EFI_CC_BLOB_GUID); 552} 553 554/* 555 * Initial set up of SNP relies on information provided by the 556 * Confidential Computing blob, which can be passed to the boot kernel 557 * by firmware/bootloader in the following ways: 558 * 559 * - via an entry in the EFI config table 560 * - via a setup_data structure, as defined by the Linux Boot Protocol 561 * 562 * Scan for the blob in that order. 563 */ 564static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp) 565{ 566 struct cc_blob_sev_info *cc_info; 567 568 cc_info = find_cc_blob_efi(bp); 569 if (cc_info) 570 goto found_cc_info; 571 572 cc_info = find_cc_blob_setup_data(bp); 573 if (!cc_info) 574 return NULL; 575 576found_cc_info: 577 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC) 578 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 579 580 return cc_info; 581} 582 583/* 584 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks 585 * will verify the SNP CPUID/MSR bits. 586 */ 587bool snp_init(struct boot_params *bp) 588{ 589 struct cc_blob_sev_info *cc_info; 590 591 if (!bp) 592 return false; 593 594 cc_info = find_cc_blob(bp); 595 if (!cc_info) 596 return false; 597 598 /* 599 * If a SNP-specific Confidential Computing blob is present, then 600 * firmware/bootloader have indicated SNP support. Verifying this 601 * involves CPUID checks which will be more reliable if the SNP 602 * CPUID table is used. See comments over snp_setup_cpuid_table() for 603 * more details. 604 */ 605 setup_cpuid_table(cc_info); 606 607 /* 608 * Pass run-time kernel a pointer to CC info via boot_params so EFI 609 * config table doesn't need to be searched again during early startup 610 * phase. 611 */ 612 bp->cc_blob_address = (u32)(unsigned long)cc_info; 613 614 return true; 615} 616 617void sev_prep_identity_maps(unsigned long top_level_pgt) 618{ 619 /* 620 * The Confidential Computing blob is used very early in uncompressed 621 * kernel to find the in-memory CPUID table to handle CPUID 622 * instructions. Make sure an identity-mapping exists so it can be 623 * accessed after switchover. 624 */ 625 if (sev_snp_enabled()) { 626 unsigned long cc_info_pa = boot_params_ptr->cc_blob_address; 627 struct cc_blob_sev_info *cc_info; 628 629 kernel_add_identity_map(cc_info_pa, cc_info_pa + sizeof(*cc_info)); 630 631 cc_info = (struct cc_blob_sev_info *)cc_info_pa; 632 kernel_add_identity_map(cc_info->cpuid_phys, cc_info->cpuid_phys + cc_info->cpuid_len); 633 } 634 635 sev_verify_cbit(top_level_pgt); 636} 637