1/* $NetBSD: patch.c,v 1.53 2022/08/20 23:48:51 riastradh Exp $ */ 2 3/*- 4 * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Patch kernel code at boot time, depending on available CPU features. 34 */ 35 36#include <sys/cdefs.h> 37__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.53 2022/08/20 23:48:51 riastradh Exp $"); 38 39#include "opt_lockdebug.h" 40#ifdef i386 41#include "opt_spldebug.h" 42#endif 43 44#include <sys/types.h> 45#include <sys/systm.h> 46 47#include <machine/cpu.h> 48#include <machine/cpufunc.h> 49#include <machine/specialreg.h> 50#include <machine/frameasm.h> 51 52#include <uvm/uvm.h> 53#include <machine/pmap.h> 54#include <machine/pmap_private.h> 55 56#include <x86/bootspace.h> 57#include <x86/cpuvar.h> 58#include <x86/cputypes.h> 59 60__link_set_decl(x86_hotpatch_descriptors, struct x86_hotpatch_descriptor); 61 62struct x86_hotpatch_destination { 63 uint8_t name; 64 uint8_t size; 65 void *addr; 66} __packed; 67 68/* -------------------------------------------------------------------------- */ 69 70/* CLAC instruction, part of SMAP. */ 71extern uint8_t hp_clac, hp_clac_end; 72static const struct x86_hotpatch_source hp_clac_source = { 73 .saddr = &hp_clac, 74 .eaddr = &hp_clac_end 75}; 76static const struct x86_hotpatch_descriptor hp_clac_desc = { 77 .name = HP_NAME_CLAC, 78 .nsrc = 1, 79 .srcs = { &hp_clac_source } 80}; 81__link_set_add_rodata(x86_hotpatch_descriptors, hp_clac_desc); 82 83/* STAC instruction, part of SMAP. */ 84extern uint8_t hp_stac, hp_stac_end; 85static const struct x86_hotpatch_source hp_stac_source = { 86 .saddr = &hp_stac, 87 .eaddr = &hp_stac_end 88}; 89static const struct x86_hotpatch_descriptor hp_stac_desc = { 90 .name = HP_NAME_STAC, 91 .nsrc = 1, 92 .srcs = { &hp_stac_source } 93}; 94__link_set_add_rodata(x86_hotpatch_descriptors, hp_stac_desc); 95 96/* Errata on certain AMD CPUs. */ 97extern uint8_t hp_retfence, hp_retfence_end; 98static const struct x86_hotpatch_source hp_retfence_source = { 99 .saddr = &hp_retfence, 100 .eaddr = &hp_retfence_end 101}; 102static const struct x86_hotpatch_descriptor hp_retfence_desc = { 103 .name = HP_NAME_RETFENCE, 104 .nsrc = 1, 105 .srcs = { &hp_retfence_source } 106}; 107__link_set_add_rodata(x86_hotpatch_descriptors, hp_retfence_desc); 108 109/* No lock when on a single processor. */ 110extern uint8_t hp_nolock, hp_nolock_end; 111static const struct x86_hotpatch_source hp_nolock_source = { 112 .saddr = &hp_nolock, 113 .eaddr = &hp_nolock_end 114}; 115static const struct x86_hotpatch_descriptor hp_nolock_desc = { 116 .name = HP_NAME_NOLOCK, 117 .nsrc = 1, 118 .srcs = { &hp_nolock_source } 119}; 120__link_set_add_rodata(x86_hotpatch_descriptors, hp_nolock_desc); 121 122#ifdef i386 123/* CAS_64. */ 124extern uint8_t _atomic_cas_cx8, _atomic_cas_cx8_end; 125static const struct x86_hotpatch_source hp_cas_cx8_source = { 126 .saddr = &_atomic_cas_cx8, 127 .eaddr = &_atomic_cas_cx8_end 128}; 129static const struct x86_hotpatch_descriptor hp_cas_cx8_desc = { 130 .name = HP_NAME_CAS_64, 131 .nsrc = 1, 132 .srcs = { &hp_cas_cx8_source } 133}; 134__link_set_add_rodata(x86_hotpatch_descriptors, hp_cas_cx8_desc); 135 136/* SPLLOWER. */ 137extern uint8_t cx8_spllower, cx8_spllower_end; 138static const struct x86_hotpatch_source hp_cx8_spllower_source = { 139 .saddr = &cx8_spllower, 140 .eaddr = &cx8_spllower_end 141}; 142static const struct x86_hotpatch_descriptor hp_cx8_spllower_desc = { 143 .name = HP_NAME_SPLLOWER, 144 .nsrc = 1, 145 .srcs = { &hp_cx8_spllower_source } 146}; 147__link_set_add_rodata(x86_hotpatch_descriptors, hp_cx8_spllower_desc); 148 149/* MUTEX_EXIT. */ 150#ifndef LOCKDEBUG 151extern uint8_t i686_mutex_spin_exit, i686_mutex_spin_exit_end; 152static const struct x86_hotpatch_source hp_i686_mutex_spin_exit_source = { 153 .saddr = &i686_mutex_spin_exit, 154 .eaddr = &i686_mutex_spin_exit_end 155}; 156static const struct x86_hotpatch_descriptor hp_i686_mutex_spin_exit_desc = { 157 .name = HP_NAME_MUTEX_EXIT, 158 .nsrc = 1, 159 .srcs = { &hp_i686_mutex_spin_exit_source } 160}; 161__link_set_add_rodata(x86_hotpatch_descriptors, hp_i686_mutex_spin_exit_desc); 162#endif 163#endif 164 165/* -------------------------------------------------------------------------- */ 166 167static inline void __unused 168patchbytes(void *addr, const uint8_t *bytes, size_t size) 169{ 170 uint8_t *ptr = (uint8_t *)addr; 171 size_t i; 172 173 for (i = 0; i < size; i++) { 174 ptr[i] = bytes[i]; 175 } 176} 177 178/* 179 * Rules: each pointer accessed in this function MUST be read-only. 180 * 181 * Called from ASM only, prototype not public. 182 */ 183int x86_hotpatch_apply(uint8_t, uint8_t); 184int 185__noubsan /* the local variables have unknown alignment to UBSan */ 186x86_hotpatch_apply(uint8_t name, uint8_t sel) 187{ 188 struct x86_hotpatch_descriptor * const *iter; 189 const struct x86_hotpatch_descriptor *desc; 190 const struct x86_hotpatch_source *src; 191 const struct x86_hotpatch_destination *hps, *hpe, *hp; 192 extern char __rodata_hotpatch_start; 193 extern char __rodata_hotpatch_end; 194 const uint8_t *bytes; 195 bool found = false; 196 size_t size; 197 198 /* 199 * Find the descriptor, and perform some sanity checks. 200 */ 201 __link_set_foreach(iter, x86_hotpatch_descriptors) { 202 desc = *iter; 203 if (desc->name == name) { 204 found = true; 205 break; 206 } 207 } 208 if (!found) 209 return -1; 210 if (desc->nsrc > 2) 211 return -1; 212 if (sel >= desc->nsrc) 213 return -1; 214 215 /* 216 * Get the hotpatch source. 217 */ 218 src = desc->srcs[sel]; 219 bytes = src->saddr; 220 size = (size_t)src->eaddr - (size_t)src->saddr; 221 222 /* 223 * Apply the hotpatch on each registered destination. 224 */ 225 hps = (struct x86_hotpatch_destination *)&__rodata_hotpatch_start; 226 hpe = (struct x86_hotpatch_destination *)&__rodata_hotpatch_end; 227 for (hp = hps; hp < hpe; hp++) { 228 if (hp->name != name) { 229 continue; 230 } 231 if (hp->size != size) { 232 return -1; 233 } 234 patchbytes(hp->addr, bytes, size); 235 } 236 237 return 0; 238} 239 240#ifdef __x86_64__ 241/* 242 * The CPU added the D bit on the text pages while we were writing to them. 243 * Remove that bit. Kinda annoying, but we can't avoid it. 244 */ 245static void 246remove_d_bit(void) 247{ 248 extern struct bootspace bootspace; 249 pt_entry_t pte; 250 vaddr_t va; 251 size_t i, n; 252 253 for (i = 0; i < BTSPACE_NSEGS; i++) { 254 if (bootspace.segs[i].type != BTSEG_TEXT) 255 continue; 256 va = bootspace.segs[i].va; 257 n = 0; 258 while (n < bootspace.segs[i].sz) { 259 if (L2_BASE[pl2_i(va)] & PTE_PS) { 260 pte = L2_BASE[pl2_i(va)] & ~PTE_D; 261 pmap_pte_set(&L2_BASE[pl2_i(va)], pte); 262 n += NBPD_L2; 263 va += NBPD_L2; 264 } else { 265 pte = L1_BASE[pl1_i(va)] & ~PTE_D; 266 pmap_pte_set(&L1_BASE[pl1_i(va)], pte); 267 n += NBPD_L1; 268 va += NBPD_L1; 269 } 270 } 271 } 272 273 tlbflushg(); 274} 275#else 276#define remove_d_bit() __nothing 277#endif 278 279/* 280 * Interrupts disabled here. Called from ASM only, prototype not public. 281 */ 282void x86_hotpatch_cleanup(int); 283void 284x86_hotpatch_cleanup(int retval) 285{ 286 if (retval != 0) { 287 panic("x86_hotpatch_apply failed"); 288 } 289 290 remove_d_bit(); 291} 292 293/* -------------------------------------------------------------------------- */ 294 295void 296x86_patch(bool early) 297{ 298 static bool first, second; 299 300 if (early) { 301 if (first) 302 return; 303 first = true; 304 } else { 305 if (second) 306 return; 307 second = true; 308 } 309 310 if (!early && ncpu == 1) { 311#ifndef LOCKDEBUG 312 /* 313 * Uniprocessor: kill LOCK prefixes. 314 */ 315 x86_hotpatch(HP_NAME_NOLOCK, 0); 316#endif 317 } 318 319#ifdef i386 320 /* 321 * Patch early and late. Second time around the 'lock' prefix 322 * may be gone. 323 */ 324 if ((cpu_feature[0] & CPUID_CX8) != 0) { 325 x86_hotpatch(HP_NAME_CAS_64, 0); 326 } 327 328#if !defined(SPLDEBUG) 329 if (!early && (cpu_feature[0] & CPUID_CX8) != 0) { 330 /* Faster splx(), mutex_spin_exit(). */ 331 x86_hotpatch(HP_NAME_SPLLOWER, 0); 332#if !defined(LOCKDEBUG) 333 x86_hotpatch(HP_NAME_MUTEX_EXIT, 0); 334#endif 335 } 336#endif /* !SPLDEBUG */ 337#endif /* i386 */ 338 339 /* 340 * On some Opteron revisions, locked operations erroneously 341 * allow memory references to be `bled' outside of critical 342 * sections. Apply workaround. 343 */ 344 if (cpu_vendor == CPUVENDOR_AMD && 345 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xe || 346 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xf && 347 CPUID_TO_EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) { 348 x86_hotpatch(HP_NAME_RETFENCE, 0); 349 } 350 351 /* 352 * If SMAP is present then patch the prepared holes with clac/stac 353 * instructions. 354 */ 355 if (!early && cpu_feature[5] & CPUID_SEF_SMAP) { 356 KASSERT(rcr4() & CR4_SMAP); 357 358 x86_hotpatch(HP_NAME_CLAC, 0); 359 x86_hotpatch(HP_NAME_STAC, 0); 360 } 361} 362