1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Handle detection, reporting and mitigation of Spectre v1, v2, v3a and v4, as 4 * detailed at: 5 * 6 * https://developer.arm.com/support/arm-security-updates/speculative-processor-vulnerability 7 * 8 * This code was originally written hastily under an awful lot of stress and so 9 * aspects of it are somewhat hacky. Unfortunately, changing anything in here 10 * instantly makes me feel ill. Thanks, Jann. Thann. 11 * 12 * Copyright (C) 2018 ARM Ltd, All Rights Reserved. 13 * Copyright (C) 2020 Google LLC 14 * 15 * "If there's something strange in your neighbourhood, who you gonna call?" 16 * 17 * Authors: Will Deacon <will@kernel.org> and Marc Zyngier <maz@kernel.org> 18 */ 19 20#include <linux/arm-smccc.h> 21#include <linux/bpf.h> 22#include <linux/cpu.h> 23#include <linux/device.h> 24#include <linux/nospec.h> 25#include <linux/prctl.h> 26#include <linux/sched/task_stack.h> 27 28#include <asm/debug-monitors.h> 29#include <asm/insn.h> 30#include <asm/spectre.h> 31#include <asm/traps.h> 32#include <asm/vectors.h> 33#include <asm/virt.h> 34 35/* 36 * We try to ensure that the mitigation state can never change as the result of 37 * onlining a late CPU. 38 */ 39static void update_mitigation_state(enum mitigation_state *oldp, 40 enum mitigation_state new) 41{ 42 enum mitigation_state state; 43 44 do { 45 state = READ_ONCE(*oldp); 46 if (new <= state) 47 break; 48 49 /* Userspace almost certainly can't deal with this. */ 50 if (WARN_ON(system_capabilities_finalized())) 51 break; 52 } while (cmpxchg_relaxed(oldp, state, new) != state); 53} 54 55/* 56 * Spectre v1. 57 * 58 * The kernel can't protect userspace for this one: it's each person for 59 * themselves. Advertise what we're doing and be done with it. 60 */ 61ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, 62 char *buf) 63{ 64 return sprintf(buf, "Mitigation: __user pointer sanitization\n"); 65} 66 67/* 68 * Spectre v2. 69 * 70 * This one sucks. A CPU is either: 71 * 72 * - Mitigated in hardware and advertised by ID_AA64PFR0_EL1.CSV2. 73 * - Mitigated in hardware and listed in our "safe list". 74 * - Mitigated in software by firmware. 75 * - Mitigated in software by a CPU-specific dance in the kernel and a 76 * firmware call at EL2. 77 * - Vulnerable. 78 * 79 * It's not unlikely for different CPUs in a big.LITTLE system to fall into 80 * different camps. 81 */ 82static enum mitigation_state spectre_v2_state; 83 84static bool __read_mostly __nospectre_v2; 85static int __init parse_spectre_v2_param(char *str) 86{ 87 __nospectre_v2 = true; 88 return 0; 89} 90early_param("nospectre_v2", parse_spectre_v2_param); 91 92static bool spectre_v2_mitigations_off(void) 93{ 94 bool ret = __nospectre_v2 || cpu_mitigations_off(); 95 96 if (ret) 97 pr_info_once("spectre-v2 mitigation disabled by command line option\n"); 98 99 return ret; 100} 101 102static const char *get_bhb_affected_string(enum mitigation_state bhb_state) 103{ 104 switch (bhb_state) { 105 case SPECTRE_UNAFFECTED: 106 return ""; 107 default: 108 case SPECTRE_VULNERABLE: 109 return ", but not BHB"; 110 case SPECTRE_MITIGATED: 111 return ", BHB"; 112 } 113} 114 115static bool _unprivileged_ebpf_enabled(void) 116{ 117#ifdef CONFIG_BPF_SYSCALL 118 return !sysctl_unprivileged_bpf_disabled; 119#else 120 return false; 121#endif 122} 123 124ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, 125 char *buf) 126{ 127 enum mitigation_state bhb_state = arm64_get_spectre_bhb_state(); 128 const char *bhb_str = get_bhb_affected_string(bhb_state); 129 const char *v2_str = "Branch predictor hardening"; 130 131 switch (spectre_v2_state) { 132 case SPECTRE_UNAFFECTED: 133 if (bhb_state == SPECTRE_UNAFFECTED) 134 return sprintf(buf, "Not affected\n"); 135 136 /* 137 * Platforms affected by Spectre-BHB can't report 138 * "Not affected" for Spectre-v2. 139 */ 140 v2_str = "CSV2"; 141 fallthrough; 142 case SPECTRE_MITIGATED: 143 if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled()) 144 return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); 145 146 return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str); 147 case SPECTRE_VULNERABLE: 148 fallthrough; 149 default: 150 return sprintf(buf, "Vulnerable\n"); 151 } 152} 153 154static enum mitigation_state spectre_v2_get_cpu_hw_mitigation_state(void) 155{ 156 u64 pfr0; 157 static const struct midr_range spectre_v2_safe_list[] = { 158 MIDR_ALL_VERSIONS(MIDR_CORTEX_A35), 159 MIDR_ALL_VERSIONS(MIDR_CORTEX_A53), 160 MIDR_ALL_VERSIONS(MIDR_CORTEX_A55), 161 MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53), 162 MIDR_ALL_VERSIONS(MIDR_HISI_TSV110), 163 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER), 164 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER), 165 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER), 166 { /* sentinel */ } 167 }; 168 169 /* If the CPU has CSV2 set, we're safe */ 170 pfr0 = read_cpuid(ID_AA64PFR0_EL1); 171 if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_CSV2_SHIFT)) 172 return SPECTRE_UNAFFECTED; 173 174 /* Alternatively, we have a list of unaffected CPUs */ 175 if (is_midr_in_range_list(read_cpuid_id(), spectre_v2_safe_list)) 176 return SPECTRE_UNAFFECTED; 177 178 return SPECTRE_VULNERABLE; 179} 180 181static enum mitigation_state spectre_v2_get_cpu_fw_mitigation_state(void) 182{ 183 int ret; 184 struct arm_smccc_res res; 185 186 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, 187 ARM_SMCCC_ARCH_WORKAROUND_1, &res); 188 189 ret = res.a0; 190 switch (ret) { 191 case SMCCC_RET_SUCCESS: 192 return SPECTRE_MITIGATED; 193 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: 194 return SPECTRE_UNAFFECTED; 195 default: 196 fallthrough; 197 case SMCCC_RET_NOT_SUPPORTED: 198 return SPECTRE_VULNERABLE; 199 } 200} 201 202bool has_spectre_v2(const struct arm64_cpu_capabilities *entry, int scope) 203{ 204 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); 205 206 if (spectre_v2_get_cpu_hw_mitigation_state() == SPECTRE_UNAFFECTED) 207 return false; 208 209 if (spectre_v2_get_cpu_fw_mitigation_state() == SPECTRE_UNAFFECTED) 210 return false; 211 212 return true; 213} 214 215enum mitigation_state arm64_get_spectre_v2_state(void) 216{ 217 return spectre_v2_state; 218} 219 220DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data); 221 222static void install_bp_hardening_cb(bp_hardening_cb_t fn) 223{ 224 __this_cpu_write(bp_hardening_data.fn, fn); 225 226 /* 227 * Vinz Clortho takes the hyp_vecs start/end "keys" at 228 * the door when we're a guest. Skip the hyp-vectors work. 229 */ 230 if (!is_hyp_mode_available()) 231 return; 232 233 __this_cpu_write(bp_hardening_data.slot, HYP_VECTOR_SPECTRE_DIRECT); 234} 235 236/* Called during entry so must be noinstr */ 237static noinstr void call_smc_arch_workaround_1(void) 238{ 239 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); 240} 241 242/* Called during entry so must be noinstr */ 243static noinstr void call_hvc_arch_workaround_1(void) 244{ 245 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); 246} 247 248/* Called during entry so must be noinstr */ 249static noinstr void qcom_link_stack_sanitisation(void) 250{ 251 u64 tmp; 252 253 asm volatile("mov %0, x30 \n" 254 ".rept 16 \n" 255 "bl . + 4 \n" 256 ".endr \n" 257 "mov x30, %0 \n" 258 : "=&r" (tmp)); 259} 260 261static bp_hardening_cb_t spectre_v2_get_sw_mitigation_cb(void) 262{ 263 u32 midr = read_cpuid_id(); 264 if (((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR) && 265 ((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR_V1)) 266 return NULL; 267 268 return qcom_link_stack_sanitisation; 269} 270 271static enum mitigation_state spectre_v2_enable_fw_mitigation(void) 272{ 273 bp_hardening_cb_t cb; 274 enum mitigation_state state; 275 276 state = spectre_v2_get_cpu_fw_mitigation_state(); 277 if (state != SPECTRE_MITIGATED) 278 return state; 279 280 if (spectre_v2_mitigations_off()) 281 return SPECTRE_VULNERABLE; 282 283 switch (arm_smccc_1_1_get_conduit()) { 284 case SMCCC_CONDUIT_HVC: 285 cb = call_hvc_arch_workaround_1; 286 break; 287 288 case SMCCC_CONDUIT_SMC: 289 cb = call_smc_arch_workaround_1; 290 break; 291 292 default: 293 return SPECTRE_VULNERABLE; 294 } 295 296 /* 297 * Prefer a CPU-specific workaround if it exists. Note that we 298 * still rely on firmware for the mitigation at EL2. 299 */ 300 cb = spectre_v2_get_sw_mitigation_cb() ?: cb; 301 install_bp_hardening_cb(cb); 302 return SPECTRE_MITIGATED; 303} 304 305void spectre_v2_enable_mitigation(const struct arm64_cpu_capabilities *__unused) 306{ 307 enum mitigation_state state; 308 309 WARN_ON(preemptible()); 310 311 state = spectre_v2_get_cpu_hw_mitigation_state(); 312 if (state == SPECTRE_VULNERABLE) 313 state = spectre_v2_enable_fw_mitigation(); 314 315 update_mitigation_state(&spectre_v2_state, state); 316} 317 318/* 319 * Spectre-v3a. 320 * 321 * Phew, there's not an awful lot to do here! We just instruct EL2 to use 322 * an indirect trampoline for the hyp vectors so that guests can't read 323 * VBAR_EL2 to defeat randomisation of the hypervisor VA layout. 324 */ 325bool has_spectre_v3a(const struct arm64_cpu_capabilities *entry, int scope) 326{ 327 static const struct midr_range spectre_v3a_unsafe_list[] = { 328 MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), 329 MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), 330 {}, 331 }; 332 333 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); 334 return is_midr_in_range_list(read_cpuid_id(), spectre_v3a_unsafe_list); 335} 336 337void spectre_v3a_enable_mitigation(const struct arm64_cpu_capabilities *__unused) 338{ 339 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); 340 341 if (this_cpu_has_cap(ARM64_SPECTRE_V3A)) 342 data->slot += HYP_VECTOR_INDIRECT; 343} 344 345/* 346 * Spectre v4. 347 * 348 * If you thought Spectre v2 was nasty, wait until you see this mess. A CPU is 349 * either: 350 * 351 * - Mitigated in hardware and listed in our "safe list". 352 * - Mitigated in hardware via PSTATE.SSBS. 353 * - Mitigated in software by firmware (sometimes referred to as SSBD). 354 * 355 * Wait, that doesn't sound so bad, does it? Keep reading... 356 * 357 * A major source of headaches is that the software mitigation is enabled both 358 * on a per-task basis, but can also be forced on for the kernel, necessitating 359 * both context-switch *and* entry/exit hooks. To make it even worse, some CPUs 360 * allow EL0 to toggle SSBS directly, which can end up with the prctl() state 361 * being stale when re-entering the kernel. The usual big.LITTLE caveats apply, 362 * so you can have systems that have both firmware and SSBS mitigations. This 363 * means we actually have to reject late onlining of CPUs with mitigations if 364 * all of the currently onlined CPUs are safelisted, as the mitigation tends to 365 * be opt-in for userspace. Yes, really, the cure is worse than the disease. 366 * 367 * The only good part is that if the firmware mitigation is present, then it is 368 * present for all CPUs, meaning we don't have to worry about late onlining of a 369 * vulnerable CPU if one of the boot CPUs is using the firmware mitigation. 370 * 371 * Give me a VAX-11/780 any day of the week... 372 */ 373static enum mitigation_state spectre_v4_state; 374 375/* This is the per-cpu state tracking whether we need to talk to firmware */ 376DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required); 377 378enum spectre_v4_policy { 379 SPECTRE_V4_POLICY_MITIGATION_DYNAMIC, 380 SPECTRE_V4_POLICY_MITIGATION_ENABLED, 381 SPECTRE_V4_POLICY_MITIGATION_DISABLED, 382}; 383 384static enum spectre_v4_policy __read_mostly __spectre_v4_policy; 385 386static const struct spectre_v4_param { 387 const char *str; 388 enum spectre_v4_policy policy; 389} spectre_v4_params[] = { 390 { "force-on", SPECTRE_V4_POLICY_MITIGATION_ENABLED, }, 391 { "force-off", SPECTRE_V4_POLICY_MITIGATION_DISABLED, }, 392 { "kernel", SPECTRE_V4_POLICY_MITIGATION_DYNAMIC, }, 393}; 394static int __init parse_spectre_v4_param(char *str) 395{ 396 int i; 397 398 if (!str || !str[0]) 399 return -EINVAL; 400 401 for (i = 0; i < ARRAY_SIZE(spectre_v4_params); i++) { 402 const struct spectre_v4_param *param = &spectre_v4_params[i]; 403 404 if (strncmp(str, param->str, strlen(param->str))) 405 continue; 406 407 __spectre_v4_policy = param->policy; 408 return 0; 409 } 410 411 return -EINVAL; 412} 413early_param("ssbd", parse_spectre_v4_param); 414 415/* 416 * Because this was all written in a rush by people working in different silos, 417 * we've ended up with multiple command line options to control the same thing. 418 * Wrap these up in some helpers, which prefer disabling the mitigation if faced 419 * with contradictory parameters. The mitigation is always either "off", 420 * "dynamic" or "on". 421 */ 422static bool spectre_v4_mitigations_off(void) 423{ 424 bool ret = cpu_mitigations_off() || 425 __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DISABLED; 426 427 if (ret) 428 pr_info_once("spectre-v4 mitigation disabled by command-line option\n"); 429 430 return ret; 431} 432 433/* Do we need to toggle the mitigation state on entry to/exit from the kernel? */ 434static bool spectre_v4_mitigations_dynamic(void) 435{ 436 return !spectre_v4_mitigations_off() && 437 __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DYNAMIC; 438} 439 440static bool spectre_v4_mitigations_on(void) 441{ 442 return !spectre_v4_mitigations_off() && 443 __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_ENABLED; 444} 445 446ssize_t cpu_show_spec_store_bypass(struct device *dev, 447 struct device_attribute *attr, char *buf) 448{ 449 switch (spectre_v4_state) { 450 case SPECTRE_UNAFFECTED: 451 return sprintf(buf, "Not affected\n"); 452 case SPECTRE_MITIGATED: 453 return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n"); 454 case SPECTRE_VULNERABLE: 455 fallthrough; 456 default: 457 return sprintf(buf, "Vulnerable\n"); 458 } 459} 460 461enum mitigation_state arm64_get_spectre_v4_state(void) 462{ 463 return spectre_v4_state; 464} 465 466static enum mitigation_state spectre_v4_get_cpu_hw_mitigation_state(void) 467{ 468 static const struct midr_range spectre_v4_safe_list[] = { 469 MIDR_ALL_VERSIONS(MIDR_CORTEX_A35), 470 MIDR_ALL_VERSIONS(MIDR_CORTEX_A53), 471 MIDR_ALL_VERSIONS(MIDR_CORTEX_A55), 472 MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53), 473 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER), 474 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER), 475 { /* sentinel */ }, 476 }; 477 478 if (is_midr_in_range_list(read_cpuid_id(), spectre_v4_safe_list)) 479 return SPECTRE_UNAFFECTED; 480 481 /* CPU features are detected first */ 482 if (this_cpu_has_cap(ARM64_SSBS)) 483 return SPECTRE_MITIGATED; 484 485 return SPECTRE_VULNERABLE; 486} 487 488static enum mitigation_state spectre_v4_get_cpu_fw_mitigation_state(void) 489{ 490 int ret; 491 struct arm_smccc_res res; 492 493 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, 494 ARM_SMCCC_ARCH_WORKAROUND_2, &res); 495 496 ret = res.a0; 497 switch (ret) { 498 case SMCCC_RET_SUCCESS: 499 return SPECTRE_MITIGATED; 500 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: 501 fallthrough; 502 case SMCCC_RET_NOT_REQUIRED: 503 return SPECTRE_UNAFFECTED; 504 default: 505 fallthrough; 506 case SMCCC_RET_NOT_SUPPORTED: 507 return SPECTRE_VULNERABLE; 508 } 509} 510 511bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope) 512{ 513 enum mitigation_state state; 514 515 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); 516 517 state = spectre_v4_get_cpu_hw_mitigation_state(); 518 if (state == SPECTRE_VULNERABLE) 519 state = spectre_v4_get_cpu_fw_mitigation_state(); 520 521 return state != SPECTRE_UNAFFECTED; 522} 523 524bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr) 525{ 526 const u32 instr_mask = ~(1U << PSTATE_Imm_shift); 527 const u32 instr_val = 0xd500401f | PSTATE_SSBS; 528 529 if ((instr & instr_mask) != instr_val) 530 return false; 531 532 if (instr & BIT(PSTATE_Imm_shift)) 533 regs->pstate |= PSR_SSBS_BIT; 534 else 535 regs->pstate &= ~PSR_SSBS_BIT; 536 537 arm64_skip_faulting_instruction(regs, 4); 538 return true; 539} 540 541static enum mitigation_state spectre_v4_enable_hw_mitigation(void) 542{ 543 enum mitigation_state state; 544 545 /* 546 * If the system is mitigated but this CPU doesn't have SSBS, then 547 * we must be on the safelist and there's nothing more to do. 548 */ 549 state = spectre_v4_get_cpu_hw_mitigation_state(); 550 if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS)) 551 return state; 552 553 if (spectre_v4_mitigations_off()) { 554 sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS); 555 set_pstate_ssbs(1); 556 return SPECTRE_VULNERABLE; 557 } 558 559 /* SCTLR_EL1.DSSBS was initialised to 0 during boot */ 560 set_pstate_ssbs(0); 561 return SPECTRE_MITIGATED; 562} 563 564/* 565 * Patch a branch over the Spectre-v4 mitigation code with a NOP so that 566 * we fallthrough and check whether firmware needs to be called on this CPU. 567 */ 568void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt, 569 __le32 *origptr, 570 __le32 *updptr, int nr_inst) 571{ 572 BUG_ON(nr_inst != 1); /* Branch -> NOP */ 573 574 if (spectre_v4_mitigations_off()) 575 return; 576 577 if (cpus_have_cap(ARM64_SSBS)) 578 return; 579 580 if (spectre_v4_mitigations_dynamic()) 581 *updptr = cpu_to_le32(aarch64_insn_gen_nop()); 582} 583 584/* 585 * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction 586 * to call into firmware to adjust the mitigation state. 587 */ 588void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt, 589 __le32 *origptr, 590 __le32 *updptr, int nr_inst) 591{ 592 u32 insn; 593 594 BUG_ON(nr_inst != 1); /* NOP -> HVC/SMC */ 595 596 switch (arm_smccc_1_1_get_conduit()) { 597 case SMCCC_CONDUIT_HVC: 598 insn = aarch64_insn_get_hvc_value(); 599 break; 600 case SMCCC_CONDUIT_SMC: 601 insn = aarch64_insn_get_smc_value(); 602 break; 603 default: 604 return; 605 } 606 607 *updptr = cpu_to_le32(insn); 608} 609 610static enum mitigation_state spectre_v4_enable_fw_mitigation(void) 611{ 612 enum mitigation_state state; 613 614 state = spectre_v4_get_cpu_fw_mitigation_state(); 615 if (state != SPECTRE_MITIGATED) 616 return state; 617 618 if (spectre_v4_mitigations_off()) { 619 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, false, NULL); 620 return SPECTRE_VULNERABLE; 621 } 622 623 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, true, NULL); 624 625 if (spectre_v4_mitigations_dynamic()) 626 __this_cpu_write(arm64_ssbd_callback_required, 1); 627 628 return SPECTRE_MITIGATED; 629} 630 631void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused) 632{ 633 enum mitigation_state state; 634 635 WARN_ON(preemptible()); 636 637 state = spectre_v4_enable_hw_mitigation(); 638 if (state == SPECTRE_VULNERABLE) 639 state = spectre_v4_enable_fw_mitigation(); 640 641 update_mitigation_state(&spectre_v4_state, state); 642} 643 644static void __update_pstate_ssbs(struct pt_regs *regs, bool state) 645{ 646 u64 bit = compat_user_mode(regs) ? PSR_AA32_SSBS_BIT : PSR_SSBS_BIT; 647 648 if (state) 649 regs->pstate |= bit; 650 else 651 regs->pstate &= ~bit; 652} 653 654void spectre_v4_enable_task_mitigation(struct task_struct *tsk) 655{ 656 struct pt_regs *regs = task_pt_regs(tsk); 657 bool ssbs = false, kthread = tsk->flags & PF_KTHREAD; 658 659 if (spectre_v4_mitigations_off()) 660 ssbs = true; 661 else if (spectre_v4_mitigations_dynamic() && !kthread) 662 ssbs = !test_tsk_thread_flag(tsk, TIF_SSBD); 663 664 __update_pstate_ssbs(regs, ssbs); 665} 666 667/* 668 * The Spectre-v4 mitigation can be controlled via a prctl() from userspace. 669 * This is interesting because the "speculation disabled" behaviour can be 670 * configured so that it is preserved across exec(), which means that the 671 * prctl() may be necessary even when PSTATE.SSBS can be toggled directly 672 * from userspace. 673 */ 674static void ssbd_prctl_enable_mitigation(struct task_struct *task) 675{ 676 task_clear_spec_ssb_noexec(task); 677 task_set_spec_ssb_disable(task); 678 set_tsk_thread_flag(task, TIF_SSBD); 679} 680 681static void ssbd_prctl_disable_mitigation(struct task_struct *task) 682{ 683 task_clear_spec_ssb_noexec(task); 684 task_clear_spec_ssb_disable(task); 685 clear_tsk_thread_flag(task, TIF_SSBD); 686} 687 688static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl) 689{ 690 switch (ctrl) { 691 case PR_SPEC_ENABLE: 692 /* Enable speculation: disable mitigation */ 693 /* 694 * Force disabled speculation prevents it from being 695 * re-enabled. 696 */ 697 if (task_spec_ssb_force_disable(task)) 698 return -EPERM; 699 700 /* 701 * If the mitigation is forced on, then speculation is forced 702 * off and we again prevent it from being re-enabled. 703 */ 704 if (spectre_v4_mitigations_on()) 705 return -EPERM; 706 707 ssbd_prctl_disable_mitigation(task); 708 break; 709 case PR_SPEC_FORCE_DISABLE: 710 /* Force disable speculation: force enable mitigation */ 711 /* 712 * If the mitigation is forced off, then speculation is forced 713 * on and we prevent it from being disabled. 714 */ 715 if (spectre_v4_mitigations_off()) 716 return -EPERM; 717 718 task_set_spec_ssb_force_disable(task); 719 fallthrough; 720 case PR_SPEC_DISABLE: 721 /* Disable speculation: enable mitigation */ 722 /* Same as PR_SPEC_FORCE_DISABLE */ 723 if (spectre_v4_mitigations_off()) 724 return -EPERM; 725 726 ssbd_prctl_enable_mitigation(task); 727 break; 728 case PR_SPEC_DISABLE_NOEXEC: 729 /* Disable speculation until execve(): enable mitigation */ 730 /* 731 * If the mitigation state is forced one way or the other, then 732 * we must fail now before we try to toggle it on execve(). 733 */ 734 if (task_spec_ssb_force_disable(task) || 735 spectre_v4_mitigations_off() || 736 spectre_v4_mitigations_on()) { 737 return -EPERM; 738 } 739 740 ssbd_prctl_enable_mitigation(task); 741 task_set_spec_ssb_noexec(task); 742 break; 743 default: 744 return -ERANGE; 745 } 746 747 spectre_v4_enable_task_mitigation(task); 748 return 0; 749} 750 751int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 752 unsigned long ctrl) 753{ 754 switch (which) { 755 case PR_SPEC_STORE_BYPASS: 756 return ssbd_prctl_set(task, ctrl); 757 default: 758 return -ENODEV; 759 } 760} 761 762static int ssbd_prctl_get(struct task_struct *task) 763{ 764 switch (spectre_v4_state) { 765 case SPECTRE_UNAFFECTED: 766 return PR_SPEC_NOT_AFFECTED; 767 case SPECTRE_MITIGATED: 768 if (spectre_v4_mitigations_on()) 769 return PR_SPEC_NOT_AFFECTED; 770 771 if (spectre_v4_mitigations_dynamic()) 772 break; 773 774 /* Mitigations are disabled, so we're vulnerable. */ 775 fallthrough; 776 case SPECTRE_VULNERABLE: 777 fallthrough; 778 default: 779 return PR_SPEC_ENABLE; 780 } 781 782 /* Check the mitigation state for this task */ 783 if (task_spec_ssb_force_disable(task)) 784 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 785 786 if (task_spec_ssb_noexec(task)) 787 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 788 789 if (task_spec_ssb_disable(task)) 790 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 791 792 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 793} 794 795int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 796{ 797 switch (which) { 798 case PR_SPEC_STORE_BYPASS: 799 return ssbd_prctl_get(task); 800 default: 801 return -ENODEV; 802 } 803} 804 805/* 806 * Spectre BHB. 807 * 808 * A CPU is either: 809 * - Mitigated by a branchy loop a CPU specific number of times, and listed 810 * in our "loop mitigated list". 811 * - Mitigated in software by the firmware Spectre v2 call. 812 * - Has the ClearBHB instruction to perform the mitigation. 813 * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no 814 * software mitigation in the vectors is needed. 815 * - Has CSV2.3, so is unaffected. 816 */ 817static enum mitigation_state spectre_bhb_state; 818 819enum mitigation_state arm64_get_spectre_bhb_state(void) 820{ 821 return spectre_bhb_state; 822} 823 824enum bhb_mitigation_bits { 825 BHB_LOOP, 826 BHB_FW, 827 BHB_HW, 828 BHB_INSN, 829}; 830static unsigned long system_bhb_mitigations; 831 832/* 833 * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any 834 * SCOPE_SYSTEM call will give the right answer. 835 */ 836u8 spectre_bhb_loop_affected(int scope) 837{ 838 u8 k = 0; 839 static u8 max_bhb_k; 840 841 if (scope == SCOPE_LOCAL_CPU) { 842 static const struct midr_range spectre_bhb_k32_list[] = { 843 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), 844 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), 845 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), 846 MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), 847 MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), 848 MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), 849 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), 850 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), 851 {}, 852 }; 853 static const struct midr_range spectre_bhb_k24_list[] = { 854 MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), 855 MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), 856 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), 857 {}, 858 }; 859 static const struct midr_range spectre_bhb_k11_list[] = { 860 MIDR_ALL_VERSIONS(MIDR_AMPERE1), 861 {}, 862 }; 863 static const struct midr_range spectre_bhb_k8_list[] = { 864 MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), 865 MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), 866 {}, 867 }; 868 869 if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list)) 870 k = 32; 871 else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list)) 872 k = 24; 873 else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k11_list)) 874 k = 11; 875 else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list)) 876 k = 8; 877 878 max_bhb_k = max(max_bhb_k, k); 879 } else { 880 k = max_bhb_k; 881 } 882 883 return k; 884} 885 886static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void) 887{ 888 int ret; 889 struct arm_smccc_res res; 890 891 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, 892 ARM_SMCCC_ARCH_WORKAROUND_3, &res); 893 894 ret = res.a0; 895 switch (ret) { 896 case SMCCC_RET_SUCCESS: 897 return SPECTRE_MITIGATED; 898 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: 899 return SPECTRE_UNAFFECTED; 900 default: 901 fallthrough; 902 case SMCCC_RET_NOT_SUPPORTED: 903 return SPECTRE_VULNERABLE; 904 } 905} 906 907static bool is_spectre_bhb_fw_affected(int scope) 908{ 909 static bool system_affected; 910 enum mitigation_state fw_state; 911 bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE; 912 static const struct midr_range spectre_bhb_firmware_mitigated_list[] = { 913 MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), 914 MIDR_ALL_VERSIONS(MIDR_CORTEX_A75), 915 {}, 916 }; 917 bool cpu_in_list = is_midr_in_range_list(read_cpuid_id(), 918 spectre_bhb_firmware_mitigated_list); 919 920 if (scope != SCOPE_LOCAL_CPU) 921 return system_affected; 922 923 fw_state = spectre_bhb_get_cpu_fw_mitigation_state(); 924 if (cpu_in_list || (has_smccc && fw_state == SPECTRE_MITIGATED)) { 925 system_affected = true; 926 return true; 927 } 928 929 return false; 930} 931 932static bool supports_ecbhb(int scope) 933{ 934 u64 mmfr1; 935 936 if (scope == SCOPE_LOCAL_CPU) 937 mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1); 938 else 939 mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); 940 941 return cpuid_feature_extract_unsigned_field(mmfr1, 942 ID_AA64MMFR1_EL1_ECBHB_SHIFT); 943} 944 945bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, 946 int scope) 947{ 948 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); 949 950 if (supports_csv2p3(scope)) 951 return false; 952 953 if (supports_clearbhb(scope)) 954 return true; 955 956 if (spectre_bhb_loop_affected(scope)) 957 return true; 958 959 if (is_spectre_bhb_fw_affected(scope)) 960 return true; 961 962 return false; 963} 964 965static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) 966{ 967 const char *v = arm64_get_bp_hardening_vector(slot); 968 969 __this_cpu_write(this_cpu_vector, v); 970 971 /* 972 * When KPTI is in use, the vectors are switched when exiting to 973 * user-space. 974 */ 975 if (cpus_have_cap(ARM64_UNMAP_KERNEL_AT_EL0)) 976 return; 977 978 write_sysreg(v, vbar_el1); 979 isb(); 980} 981 982static bool __read_mostly __nospectre_bhb; 983static int __init parse_spectre_bhb_param(char *str) 984{ 985 __nospectre_bhb = true; 986 return 0; 987} 988early_param("nospectre_bhb", parse_spectre_bhb_param); 989 990void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) 991{ 992 bp_hardening_cb_t cpu_cb; 993 enum mitigation_state fw_state, state = SPECTRE_VULNERABLE; 994 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); 995 996 if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU)) 997 return; 998 999 if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) { 1000 /* No point mitigating Spectre-BHB alone. */ 1001 } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { 1002 pr_info_once("spectre-bhb mitigation disabled by compile time option\n"); 1003 } else if (cpu_mitigations_off() || __nospectre_bhb) { 1004 pr_info_once("spectre-bhb mitigation disabled by command line option\n"); 1005 } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { 1006 state = SPECTRE_MITIGATED; 1007 set_bit(BHB_HW, &system_bhb_mitigations); 1008 } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) { 1009 /* 1010 * Ensure KVM uses the indirect vector which will have ClearBHB 1011 * added. 1012 */ 1013 if (!data->slot) 1014 data->slot = HYP_VECTOR_INDIRECT; 1015 1016 this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN); 1017 state = SPECTRE_MITIGATED; 1018 set_bit(BHB_INSN, &system_bhb_mitigations); 1019 } else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) { 1020 /* 1021 * Ensure KVM uses the indirect vector which will have the 1022 * branchy-loop added. A57/A72-r0 will already have selected 1023 * the spectre-indirect vector, which is sufficient for BHB 1024 * too. 1025 */ 1026 if (!data->slot) 1027 data->slot = HYP_VECTOR_INDIRECT; 1028 1029 this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP); 1030 state = SPECTRE_MITIGATED; 1031 set_bit(BHB_LOOP, &system_bhb_mitigations); 1032 } else if (is_spectre_bhb_fw_affected(SCOPE_LOCAL_CPU)) { 1033 fw_state = spectre_bhb_get_cpu_fw_mitigation_state(); 1034 if (fw_state == SPECTRE_MITIGATED) { 1035 /* 1036 * Ensure KVM uses one of the spectre bp_hardening 1037 * vectors. The indirect vector doesn't include the EL3 1038 * call, so needs upgrading to 1039 * HYP_VECTOR_SPECTRE_INDIRECT. 1040 */ 1041 if (!data->slot || data->slot == HYP_VECTOR_INDIRECT) 1042 data->slot += 1; 1043 1044 this_cpu_set_vectors(EL1_VECTOR_BHB_FW); 1045 1046 /* 1047 * The WA3 call in the vectors supersedes the WA1 call 1048 * made during context-switch. Uninstall any firmware 1049 * bp_hardening callback. 1050 */ 1051 cpu_cb = spectre_v2_get_sw_mitigation_cb(); 1052 if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb) 1053 __this_cpu_write(bp_hardening_data.fn, NULL); 1054 1055 state = SPECTRE_MITIGATED; 1056 set_bit(BHB_FW, &system_bhb_mitigations); 1057 } 1058 } 1059 1060 update_mitigation_state(&spectre_bhb_state, state); 1061} 1062 1063/* Patched to NOP when enabled */ 1064void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt, 1065 __le32 *origptr, 1066 __le32 *updptr, int nr_inst) 1067{ 1068 BUG_ON(nr_inst != 1); 1069 1070 if (test_bit(BHB_LOOP, &system_bhb_mitigations)) 1071 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); 1072} 1073 1074/* Patched to NOP when enabled */ 1075void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt, 1076 __le32 *origptr, 1077 __le32 *updptr, int nr_inst) 1078{ 1079 BUG_ON(nr_inst != 1); 1080 1081 if (test_bit(BHB_FW, &system_bhb_mitigations)) 1082 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); 1083} 1084 1085/* Patched to correct the immediate */ 1086void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt, 1087 __le32 *origptr, __le32 *updptr, int nr_inst) 1088{ 1089 u8 rd; 1090 u32 insn; 1091 u16 loop_count = spectre_bhb_loop_affected(SCOPE_SYSTEM); 1092 1093 BUG_ON(nr_inst != 1); /* MOV -> MOV */ 1094 1095 if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) 1096 return; 1097 1098 insn = le32_to_cpu(*origptr); 1099 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn); 1100 insn = aarch64_insn_gen_movewide(rd, loop_count, 0, 1101 AARCH64_INSN_VARIANT_64BIT, 1102 AARCH64_INSN_MOVEWIDE_ZERO); 1103 *updptr++ = cpu_to_le32(insn); 1104} 1105 1106/* Patched to mov WA3 when supported */ 1107void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt, 1108 __le32 *origptr, __le32 *updptr, int nr_inst) 1109{ 1110 u8 rd; 1111 u32 insn; 1112 1113 BUG_ON(nr_inst != 1); /* MOV -> MOV */ 1114 1115 if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) || 1116 !test_bit(BHB_FW, &system_bhb_mitigations)) 1117 return; 1118 1119 insn = le32_to_cpu(*origptr); 1120 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn); 1121 1122 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR, 1123 AARCH64_INSN_VARIANT_32BIT, 1124 AARCH64_INSN_REG_ZR, rd, 1125 ARM_SMCCC_ARCH_WORKAROUND_3); 1126 if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT)) 1127 return; 1128 1129 *updptr++ = cpu_to_le32(insn); 1130} 1131 1132/* Patched to NOP when not supported */ 1133void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt, 1134 __le32 *origptr, __le32 *updptr, int nr_inst) 1135{ 1136 BUG_ON(nr_inst != 2); 1137 1138 if (test_bit(BHB_INSN, &system_bhb_mitigations)) 1139 return; 1140 1141 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); 1142 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); 1143} 1144 1145#ifdef CONFIG_BPF_SYSCALL 1146#define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n" 1147void unpriv_ebpf_notify(int new_state) 1148{ 1149 if (spectre_v2_state == SPECTRE_VULNERABLE || 1150 spectre_bhb_state != SPECTRE_MITIGATED) 1151 return; 1152 1153 if (!new_state) 1154 pr_err("WARNING: %s", EBPF_WARN); 1155} 1156#endif 1157