1/*	$NetBSD: patch.c,v 1.53 2022/08/20 23:48:51 riastradh Exp $	*/
2
3/*-
4 * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Patch kernel code at boot time, depending on available CPU features.
34 */
35
36#include <sys/cdefs.h>
37__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.53 2022/08/20 23:48:51 riastradh Exp $");
38
39#include "opt_lockdebug.h"
40#ifdef i386
41#include "opt_spldebug.h"
42#endif
43
44#include <sys/types.h>
45#include <sys/systm.h>
46
47#include <machine/cpu.h>
48#include <machine/cpufunc.h>
49#include <machine/specialreg.h>
50#include <machine/frameasm.h>
51
52#include <uvm/uvm.h>
53#include <machine/pmap.h>
54#include <machine/pmap_private.h>
55
56#include <x86/bootspace.h>
57#include <x86/cpuvar.h>
58#include <x86/cputypes.h>
59
60__link_set_decl(x86_hotpatch_descriptors, struct x86_hotpatch_descriptor);
61
62struct x86_hotpatch_destination {
63	uint8_t name;
64	uint8_t size;
65	void *addr;
66} __packed;
67
68/* -------------------------------------------------------------------------- */
69
70/* CLAC instruction, part of SMAP. */
71extern uint8_t hp_clac, hp_clac_end;
72static const struct x86_hotpatch_source hp_clac_source = {
73	.saddr = &hp_clac,
74	.eaddr = &hp_clac_end
75};
76static const struct x86_hotpatch_descriptor hp_clac_desc = {
77	.name = HP_NAME_CLAC,
78	.nsrc = 1,
79	.srcs = { &hp_clac_source }
80};
81__link_set_add_rodata(x86_hotpatch_descriptors, hp_clac_desc);
82
83/* STAC instruction, part of SMAP. */
84extern uint8_t hp_stac, hp_stac_end;
85static const struct x86_hotpatch_source hp_stac_source = {
86	.saddr = &hp_stac,
87	.eaddr = &hp_stac_end
88};
89static const struct x86_hotpatch_descriptor hp_stac_desc = {
90	.name = HP_NAME_STAC,
91	.nsrc = 1,
92	.srcs = { &hp_stac_source }
93};
94__link_set_add_rodata(x86_hotpatch_descriptors, hp_stac_desc);
95
96/* Errata on certain AMD CPUs. */
97extern uint8_t hp_retfence, hp_retfence_end;
98static const struct x86_hotpatch_source hp_retfence_source = {
99	.saddr = &hp_retfence,
100	.eaddr = &hp_retfence_end
101};
102static const struct x86_hotpatch_descriptor hp_retfence_desc = {
103	.name = HP_NAME_RETFENCE,
104	.nsrc = 1,
105	.srcs = { &hp_retfence_source }
106};
107__link_set_add_rodata(x86_hotpatch_descriptors, hp_retfence_desc);
108
109/* No lock when on a single processor. */
110extern uint8_t hp_nolock, hp_nolock_end;
111static const struct x86_hotpatch_source hp_nolock_source = {
112	.saddr = &hp_nolock,
113	.eaddr = &hp_nolock_end
114};
115static const struct x86_hotpatch_descriptor hp_nolock_desc = {
116	.name = HP_NAME_NOLOCK,
117	.nsrc = 1,
118	.srcs = { &hp_nolock_source }
119};
120__link_set_add_rodata(x86_hotpatch_descriptors, hp_nolock_desc);
121
122#ifdef i386
123/* CAS_64. */
124extern uint8_t _atomic_cas_cx8, _atomic_cas_cx8_end;
125static const struct x86_hotpatch_source hp_cas_cx8_source = {
126	.saddr = &_atomic_cas_cx8,
127	.eaddr = &_atomic_cas_cx8_end
128};
129static const struct x86_hotpatch_descriptor hp_cas_cx8_desc = {
130	.name = HP_NAME_CAS_64,
131	.nsrc = 1,
132	.srcs = { &hp_cas_cx8_source }
133};
134__link_set_add_rodata(x86_hotpatch_descriptors, hp_cas_cx8_desc);
135
136/* SPLLOWER. */
137extern uint8_t cx8_spllower, cx8_spllower_end;
138static const struct x86_hotpatch_source hp_cx8_spllower_source = {
139	.saddr = &cx8_spllower,
140	.eaddr = &cx8_spllower_end
141};
142static const struct x86_hotpatch_descriptor hp_cx8_spllower_desc = {
143	.name = HP_NAME_SPLLOWER,
144	.nsrc = 1,
145	.srcs = { &hp_cx8_spllower_source }
146};
147__link_set_add_rodata(x86_hotpatch_descriptors, hp_cx8_spllower_desc);
148
149/* MUTEX_EXIT. */
150#ifndef LOCKDEBUG
151extern uint8_t i686_mutex_spin_exit, i686_mutex_spin_exit_end;
152static const struct x86_hotpatch_source hp_i686_mutex_spin_exit_source = {
153	.saddr = &i686_mutex_spin_exit,
154	.eaddr = &i686_mutex_spin_exit_end
155};
156static const struct x86_hotpatch_descriptor hp_i686_mutex_spin_exit_desc = {
157	.name = HP_NAME_MUTEX_EXIT,
158	.nsrc = 1,
159	.srcs = { &hp_i686_mutex_spin_exit_source }
160};
161__link_set_add_rodata(x86_hotpatch_descriptors, hp_i686_mutex_spin_exit_desc);
162#endif
163#endif
164
165/* -------------------------------------------------------------------------- */
166
167static inline void __unused
168patchbytes(void *addr, const uint8_t *bytes, size_t size)
169{
170	uint8_t *ptr = (uint8_t *)addr;
171	size_t i;
172
173	for (i = 0; i < size; i++) {
174		ptr[i] = bytes[i];
175	}
176}
177
178/*
179 * Rules: each pointer accessed in this function MUST be read-only.
180 *
181 * Called from ASM only, prototype not public.
182 */
183int x86_hotpatch_apply(uint8_t, uint8_t);
184int
185__noubsan /* the local variables have unknown alignment to UBSan */
186x86_hotpatch_apply(uint8_t name, uint8_t sel)
187{
188	struct x86_hotpatch_descriptor * const *iter;
189	const struct x86_hotpatch_descriptor *desc;
190	const struct x86_hotpatch_source *src;
191	const struct x86_hotpatch_destination *hps, *hpe, *hp;
192	extern char __rodata_hotpatch_start;
193	extern char __rodata_hotpatch_end;
194	const uint8_t *bytes;
195	bool found = false;
196	size_t size;
197
198	/*
199	 * Find the descriptor, and perform some sanity checks.
200	 */
201	__link_set_foreach(iter, x86_hotpatch_descriptors) {
202		desc = *iter;
203		if (desc->name == name) {
204			found = true;
205			break;
206		}
207	}
208	if (!found)
209		return -1;
210	if (desc->nsrc > 2)
211		return -1;
212	if (sel >= desc->nsrc)
213		return -1;
214
215	/*
216	 * Get the hotpatch source.
217	 */
218	src = desc->srcs[sel];
219	bytes = src->saddr;
220	size = (size_t)src->eaddr - (size_t)src->saddr;
221
222	/*
223	 * Apply the hotpatch on each registered destination.
224	 */
225	hps = (struct x86_hotpatch_destination *)&__rodata_hotpatch_start;
226	hpe = (struct x86_hotpatch_destination *)&__rodata_hotpatch_end;
227	for (hp = hps; hp < hpe; hp++) {
228		if (hp->name != name) {
229			continue;
230		}
231		if (hp->size != size) {
232			return -1;
233		}
234		patchbytes(hp->addr, bytes, size);
235	}
236
237	return 0;
238}
239
240#ifdef __x86_64__
241/*
242 * The CPU added the D bit on the text pages while we were writing to them.
243 * Remove that bit. Kinda annoying, but we can't avoid it.
244 */
245static void
246remove_d_bit(void)
247{
248	extern struct bootspace bootspace;
249	pt_entry_t pte;
250	vaddr_t va;
251	size_t i, n;
252
253	for (i = 0; i < BTSPACE_NSEGS; i++) {
254		if (bootspace.segs[i].type != BTSEG_TEXT)
255			continue;
256		va = bootspace.segs[i].va;
257		n = 0;
258		while (n < bootspace.segs[i].sz) {
259			if (L2_BASE[pl2_i(va)] & PTE_PS) {
260				pte = L2_BASE[pl2_i(va)] & ~PTE_D;
261				pmap_pte_set(&L2_BASE[pl2_i(va)], pte);
262				n += NBPD_L2;
263				va += NBPD_L2;
264			} else {
265				pte = L1_BASE[pl1_i(va)] & ~PTE_D;
266				pmap_pte_set(&L1_BASE[pl1_i(va)], pte);
267				n += NBPD_L1;
268				va += NBPD_L1;
269			}
270		}
271	}
272
273	tlbflushg();
274}
275#else
276#define remove_d_bit()	__nothing
277#endif
278
279/*
280 * Interrupts disabled here. Called from ASM only, prototype not public.
281 */
282void x86_hotpatch_cleanup(int);
283void
284x86_hotpatch_cleanup(int retval)
285{
286	if (retval != 0) {
287		panic("x86_hotpatch_apply failed");
288	}
289
290	remove_d_bit();
291}
292
293/* -------------------------------------------------------------------------- */
294
295void
296x86_patch(bool early)
297{
298	static bool first, second;
299
300	if (early) {
301		if (first)
302			return;
303		first = true;
304	} else {
305		if (second)
306			return;
307		second = true;
308	}
309
310	if (!early && ncpu == 1) {
311#ifndef LOCKDEBUG
312		/*
313		 * Uniprocessor: kill LOCK prefixes.
314		 */
315		x86_hotpatch(HP_NAME_NOLOCK, 0);
316#endif
317	}
318
319#ifdef i386
320	/*
321	 * Patch early and late.  Second time around the 'lock' prefix
322	 * may be gone.
323	 */
324	if ((cpu_feature[0] & CPUID_CX8) != 0) {
325		x86_hotpatch(HP_NAME_CAS_64, 0);
326	}
327
328#if !defined(SPLDEBUG)
329	if (!early && (cpu_feature[0] & CPUID_CX8) != 0) {
330		/* Faster splx(), mutex_spin_exit(). */
331		x86_hotpatch(HP_NAME_SPLLOWER, 0);
332#if !defined(LOCKDEBUG)
333		x86_hotpatch(HP_NAME_MUTEX_EXIT, 0);
334#endif
335	}
336#endif /* !SPLDEBUG */
337#endif	/* i386 */
338
339	/*
340	 * On some Opteron revisions, locked operations erroneously
341	 * allow memory references to be `bled' outside of critical
342	 * sections.  Apply workaround.
343	 */
344	if (cpu_vendor == CPUVENDOR_AMD &&
345	    (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xe ||
346	    (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xf &&
347	    CPUID_TO_EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) {
348		x86_hotpatch(HP_NAME_RETFENCE, 0);
349	}
350
351	/*
352	 * If SMAP is present then patch the prepared holes with clac/stac
353	 * instructions.
354	 */
355	if (!early && cpu_feature[5] & CPUID_SEF_SMAP) {
356		KASSERT(rcr4() & CR4_SMAP);
357
358		x86_hotpatch(HP_NAME_CLAC, 0);
359		x86_hotpatch(HP_NAME_STAC, 0);
360	}
361}
362