1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Intel Transactional Synchronization Extensions (TSX) control. 4 * 5 * Copyright (C) 2019-2021 Intel Corporation 6 * 7 * Author: 8 * Pawan Gupta <pawan.kumar.gupta@linux.intel.com> 9 */ 10 11#include <linux/cpufeature.h> 12 13#include <asm/cmdline.h> 14#include <asm/cpu.h> 15 16#include "cpu.h" 17 18#undef pr_fmt 19#define pr_fmt(fmt) "tsx: " fmt 20 21enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED; 22 23static void tsx_disable(void) 24{ 25 u64 tsx; 26 27 rdmsrl(MSR_IA32_TSX_CTRL, tsx); 28 29 /* Force all transactions to immediately abort */ 30 tsx |= TSX_CTRL_RTM_DISABLE; 31 32 /* 33 * Ensure TSX support is not enumerated in CPUID. 34 * This is visible to userspace and will ensure they 35 * do not waste resources trying TSX transactions that 36 * will always abort. 37 */ 38 tsx |= TSX_CTRL_CPUID_CLEAR; 39 40 wrmsrl(MSR_IA32_TSX_CTRL, tsx); 41} 42 43static void tsx_enable(void) 44{ 45 u64 tsx; 46 47 rdmsrl(MSR_IA32_TSX_CTRL, tsx); 48 49 /* Enable the RTM feature in the cpu */ 50 tsx &= ~TSX_CTRL_RTM_DISABLE; 51 52 /* 53 * Ensure TSX support is enumerated in CPUID. 54 * This is visible to userspace and will ensure they 55 * can enumerate and use the TSX feature. 56 */ 57 tsx &= ~TSX_CTRL_CPUID_CLEAR; 58 59 wrmsrl(MSR_IA32_TSX_CTRL, tsx); 60} 61 62static enum tsx_ctrl_states x86_get_tsx_auto_mode(void) 63{ 64 if (boot_cpu_has_bug(X86_BUG_TAA)) 65 return TSX_CTRL_DISABLE; 66 67 return TSX_CTRL_ENABLE; 68} 69 70/* 71 * Disabling TSX is not a trivial business. 72 * 73 * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT 74 * which says that TSX is practically disabled (all transactions are 75 * aborted by default). When that bit is set, the kernel unconditionally 76 * disables TSX. 77 * 78 * In order to do that, however, it needs to dance a bit: 79 * 80 * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and 81 * the MSR is present only when *two* CPUID bits are set: 82 * 83 * - X86_FEATURE_RTM_ALWAYS_ABORT 84 * - X86_FEATURE_TSX_FORCE_ABORT 85 * 86 * 2. The second method is for CPUs which do not have the above-mentioned 87 * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX 88 * through that one. Those CPUs can also have the initially mentioned 89 * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy 90 * applies: TSX gets disabled unconditionally. 91 * 92 * When either of the two methods are present, the kernel disables TSX and 93 * clears the respective RTM and HLE feature flags. 94 * 95 * An additional twist in the whole thing presents late microcode loading 96 * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID 97 * bit to be set after the update. 98 * 99 * A subsequent hotplug operation on any logical CPU except the BSP will 100 * cause for the supported CPUID feature bits to get re-detected and, if 101 * RTM and HLE get cleared all of a sudden, but, userspace did consult 102 * them before the update, then funny explosions will happen. Long story 103 * short: the kernel doesn't modify CPUID feature bits after booting. 104 * 105 * That's why, this function's call in init_intel() doesn't clear the 106 * feature flags. 107 */ 108static void tsx_clear_cpuid(void) 109{ 110 u64 msr; 111 112 /* 113 * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID 114 * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present. 115 */ 116 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) && 117 boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) { 118 rdmsrl(MSR_TSX_FORCE_ABORT, msr); 119 msr |= MSR_TFA_TSX_CPUID_CLEAR; 120 wrmsrl(MSR_TSX_FORCE_ABORT, msr); 121 } else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) { 122 rdmsrl(MSR_IA32_TSX_CTRL, msr); 123 msr |= TSX_CTRL_CPUID_CLEAR; 124 wrmsrl(MSR_IA32_TSX_CTRL, msr); 125 } 126} 127 128/* 129 * Disable TSX development mode 130 * 131 * When the microcode released in Feb 2022 is applied, TSX will be disabled by 132 * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123 133 * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is 134 * not recommended for production deployments. In particular, applying MD_CLEAR 135 * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient 136 * execution attack may not be effective on these processors when Intel TSX is 137 * enabled with updated microcode. 138 */ 139static void tsx_dev_mode_disable(void) 140{ 141 u64 mcu_opt_ctrl; 142 143 /* Check if RTM_ALLOW exists */ 144 if (!boot_cpu_has_bug(X86_BUG_TAA) || 145 !cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL) || 146 !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL)) 147 return; 148 149 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); 150 151 if (mcu_opt_ctrl & RTM_ALLOW) { 152 mcu_opt_ctrl &= ~RTM_ALLOW; 153 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); 154 setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT); 155 } 156} 157 158void __init tsx_init(void) 159{ 160 char arg[5] = {}; 161 int ret; 162 163 tsx_dev_mode_disable(); 164 165 /* 166 * Hardware will always abort a TSX transaction when the CPUID bit 167 * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate 168 * CPUID.RTM and CPUID.HLE bits. Clear them here. 169 */ 170 if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) { 171 tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT; 172 tsx_clear_cpuid(); 173 setup_clear_cpu_cap(X86_FEATURE_RTM); 174 setup_clear_cpu_cap(X86_FEATURE_HLE); 175 return; 176 } 177 178 /* 179 * TSX is controlled via MSR_IA32_TSX_CTRL. However, support for this 180 * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES. 181 * 182 * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a 183 * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES 184 * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get 185 * MSR_IA32_TSX_CTRL support even after a microcode update. Thus, 186 * tsx= cmdline requests will do nothing on CPUs without 187 * MSR_IA32_TSX_CTRL support. 188 */ 189 if (x86_read_arch_cap_msr() & ARCH_CAP_TSX_CTRL_MSR) { 190 setup_force_cpu_cap(X86_FEATURE_MSR_TSX_CTRL); 191 } else { 192 tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED; 193 return; 194 } 195 196 ret = cmdline_find_option(boot_command_line, "tsx", arg, sizeof(arg)); 197 if (ret >= 0) { 198 if (!strcmp(arg, "on")) { 199 tsx_ctrl_state = TSX_CTRL_ENABLE; 200 } else if (!strcmp(arg, "off")) { 201 tsx_ctrl_state = TSX_CTRL_DISABLE; 202 } else if (!strcmp(arg, "auto")) { 203 tsx_ctrl_state = x86_get_tsx_auto_mode(); 204 } else { 205 tsx_ctrl_state = TSX_CTRL_DISABLE; 206 pr_err("invalid option, defaulting to off\n"); 207 } 208 } else { 209 /* tsx= not provided */ 210 if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO)) 211 tsx_ctrl_state = x86_get_tsx_auto_mode(); 212 else if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF)) 213 tsx_ctrl_state = TSX_CTRL_DISABLE; 214 else 215 tsx_ctrl_state = TSX_CTRL_ENABLE; 216 } 217 218 if (tsx_ctrl_state == TSX_CTRL_DISABLE) { 219 tsx_disable(); 220 221 /* 222 * tsx_disable() will change the state of the RTM and HLE CPUID 223 * bits. Clear them here since they are now expected to be not 224 * set. 225 */ 226 setup_clear_cpu_cap(X86_FEATURE_RTM); 227 setup_clear_cpu_cap(X86_FEATURE_HLE); 228 } else if (tsx_ctrl_state == TSX_CTRL_ENABLE) { 229 230 /* 231 * HW defaults TSX to be enabled at bootup. 232 * We may still need the TSX enable support 233 * during init for special cases like 234 * kexec after TSX is disabled. 235 */ 236 tsx_enable(); 237 238 /* 239 * tsx_enable() will change the state of the RTM and HLE CPUID 240 * bits. Force them here since they are now expected to be set. 241 */ 242 setup_force_cpu_cap(X86_FEATURE_RTM); 243 setup_force_cpu_cap(X86_FEATURE_HLE); 244 } 245} 246 247void tsx_ap_init(void) 248{ 249 tsx_dev_mode_disable(); 250 251 if (tsx_ctrl_state == TSX_CTRL_ENABLE) 252 tsx_enable(); 253 else if (tsx_ctrl_state == TSX_CTRL_DISABLE) 254 tsx_disable(); 255 else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT) 256 /* See comment over that function for more details. */ 257 tsx_clear_cpuid(); 258} 259