1// SPDX-License-Identifier: GPL-2.0
2#include "misc.h"
3#include <asm/bootparam.h>
4#include <asm/e820/types.h>
5#include <asm/processor.h>
6#include "pgtable.h"
7#include "../string.h"
8#include "efi.h"
9
10#define BIOS_START_MIN		0x20000U	/* 128K, less than this is insane */
11#define BIOS_START_MAX		0x9f000U	/* 640K, absolute maximum */
12
13#ifdef CONFIG_X86_5LEVEL
14/* __pgtable_l5_enabled needs to be in .data to avoid being cleared along with .bss */
15unsigned int __section(".data") __pgtable_l5_enabled;
16unsigned int __section(".data") pgdir_shift = 39;
17unsigned int __section(".data") ptrs_per_p4d = 1;
18#endif
19
20/* Buffer to preserve trampoline memory */
21static char trampoline_save[TRAMPOLINE_32BIT_SIZE];
22
23/*
24 * Trampoline address will be printed by extract_kernel() for debugging
25 * purposes.
26 *
27 * Avoid putting the pointer into .bss as it will be cleared between
28 * configure_5level_paging() and extract_kernel().
29 */
30unsigned long *trampoline_32bit __section(".data");
31
32int cmdline_find_option_bool(const char *option);
33
34static unsigned long find_trampoline_placement(void)
35{
36	unsigned long bios_start = 0, ebda_start = 0;
37	struct boot_e820_entry *entry;
38	char *signature;
39	int i;
40
41	/*
42	 * Find a suitable spot for the trampoline.
43	 * This code is based on reserve_bios_regions().
44	 */
45
46	/*
47	 * EFI systems may not provide legacy ROM. The memory may not be mapped
48	 * at all.
49	 *
50	 * Only look for values in the legacy ROM for non-EFI system.
51	 */
52	signature = (char *)&boot_params_ptr->efi_info.efi_loader_signature;
53	if (strncmp(signature, EFI32_LOADER_SIGNATURE, 4) &&
54	    strncmp(signature, EFI64_LOADER_SIGNATURE, 4)) {
55		ebda_start = *(unsigned short *)0x40e << 4;
56		bios_start = *(unsigned short *)0x413 << 10;
57	}
58
59	if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX)
60		bios_start = BIOS_START_MAX;
61
62	if (ebda_start > BIOS_START_MIN && ebda_start < bios_start)
63		bios_start = ebda_start;
64
65	bios_start = round_down(bios_start, PAGE_SIZE);
66
67	/* Find the first usable memory region under bios_start. */
68	for (i = boot_params_ptr->e820_entries - 1; i >= 0; i--) {
69		unsigned long new = bios_start;
70
71		entry = &boot_params_ptr->e820_table[i];
72
73		/* Skip all entries above bios_start. */
74		if (bios_start <= entry->addr)
75			continue;
76
77		/* Skip non-RAM entries. */
78		if (entry->type != E820_TYPE_RAM)
79			continue;
80
81		/* Adjust bios_start to the end of the entry if needed. */
82		if (bios_start > entry->addr + entry->size)
83			new = entry->addr + entry->size;
84
85		/* Keep bios_start page-aligned. */
86		new = round_down(new, PAGE_SIZE);
87
88		/* Skip the entry if it's too small. */
89		if (new - TRAMPOLINE_32BIT_SIZE < entry->addr)
90			continue;
91
92		/* Protect against underflow. */
93		if (new - TRAMPOLINE_32BIT_SIZE > bios_start)
94			break;
95
96		bios_start = new;
97		break;
98	}
99
100	/* Place the trampoline just below the end of low memory */
101	return bios_start - TRAMPOLINE_32BIT_SIZE;
102}
103
104asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable)
105{
106	void (*toggle_la57)(void *cr3);
107	bool l5_required = false;
108
109	/* Initialize boot_params. Required for cmdline_find_option_bool(). */
110	boot_params_ptr = bp;
111
112	/*
113	 * Check if LA57 is desired and supported.
114	 *
115	 * There are several parts to the check:
116	 *   - if the kernel supports 5-level paging: CONFIG_X86_5LEVEL=y
117	 *   - if user asked to disable 5-level paging: no5lvl in cmdline
118	 *   - if the machine supports 5-level paging:
119	 *     + CPUID leaf 7 is supported
120	 *     + the leaf has the feature bit set
121	 *
122	 * That's substitute for boot_cpu_has() in early boot code.
123	 */
124	if (IS_ENABLED(CONFIG_X86_5LEVEL) &&
125			!cmdline_find_option_bool("no5lvl") &&
126			native_cpuid_eax(0) >= 7 &&
127			(native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) {
128		l5_required = true;
129
130		/* Initialize variables for 5-level paging */
131		__pgtable_l5_enabled = 1;
132		pgdir_shift = 48;
133		ptrs_per_p4d = 512;
134	}
135
136	/*
137	 * The trampoline will not be used if the paging mode is already set to
138	 * the desired one.
139	 */
140	if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))
141		return;
142
143	trampoline_32bit = (unsigned long *)find_trampoline_placement();
144
145	/* Preserve trampoline memory */
146	memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE);
147
148	/* Clear trampoline memory first */
149	memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);
150
151	/* Copy trampoline code in place */
152	toggle_la57 = memcpy(trampoline_32bit +
153			TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
154			&trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
155
156	/*
157	 * Avoid the need for a stack in the 32-bit trampoline code, by using
158	 * LJMP rather than LRET to return back to long mode. LJMP takes an
159	 * immediate absolute address, which needs to be adjusted based on the
160	 * placement of the trampoline.
161	 */
162	*(u32 *)((u8 *)toggle_la57 + trampoline_ljmp_imm_offset) +=
163						(unsigned long)toggle_la57;
164
165	/*
166	 * The code below prepares page table in trampoline memory.
167	 *
168	 * The new page table will be used by trampoline code for switching
169	 * from 4- to 5-level paging or vice versa.
170	 */
171
172	if (l5_required) {
173		/*
174		 * For 4- to 5-level paging transition, set up current CR3 as
175		 * the first and the only entry in a new top-level page table.
176		 */
177		*trampoline_32bit = __native_read_cr3() | _PAGE_TABLE_NOENC;
178	} else {
179		unsigned long src;
180
181		/*
182		 * For 5- to 4-level paging transition, copy page table pointed
183		 * by first entry in the current top-level page table as our
184		 * new top-level page table.
185		 *
186		 * We cannot just point to the page table from trampoline as it
187		 * may be above 4G.
188		 */
189		src = *(unsigned long *)__native_read_cr3() & PAGE_MASK;
190		memcpy(trampoline_32bit, (void *)src, PAGE_SIZE);
191	}
192
193	toggle_la57(trampoline_32bit);
194
195	/*
196	 * Move the top level page table out of trampoline memory.
197	 */
198	memcpy(pgtable, trampoline_32bit, PAGE_SIZE);
199	native_write_cr3((unsigned long)pgtable);
200
201	/* Restore trampoline memory */
202	memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
203}
204