1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * alternative runtime patching
4 * inspired by the ARM64 and x86 version
5 *
6 * Copyright (C) 2021 Sifive.
7 */
8
9#include <linux/init.h>
10#include <linux/module.h>
11#include <linux/cpu.h>
12#include <linux/uaccess.h>
13#include <asm/alternative.h>
14#include <asm/module.h>
15#include <asm/sections.h>
16#include <asm/vdso.h>
17#include <asm/vendorid_list.h>
18#include <asm/sbi.h>
19#include <asm/csr.h>
20#include <asm/insn.h>
21#include <asm/patch.h>
22
23struct cpu_manufacturer_info_t {
24	unsigned long vendor_id;
25	unsigned long arch_id;
26	unsigned long imp_id;
27	void (*patch_func)(struct alt_entry *begin, struct alt_entry *end,
28				  unsigned long archid, unsigned long impid,
29				  unsigned int stage);
30};
31
32static void riscv_fill_cpu_mfr_info(struct cpu_manufacturer_info_t *cpu_mfr_info)
33{
34#ifdef CONFIG_RISCV_M_MODE
35	cpu_mfr_info->vendor_id = csr_read(CSR_MVENDORID);
36	cpu_mfr_info->arch_id = csr_read(CSR_MARCHID);
37	cpu_mfr_info->imp_id = csr_read(CSR_MIMPID);
38#else
39	cpu_mfr_info->vendor_id = sbi_get_mvendorid();
40	cpu_mfr_info->arch_id = sbi_get_marchid();
41	cpu_mfr_info->imp_id = sbi_get_mimpid();
42#endif
43
44	switch (cpu_mfr_info->vendor_id) {
45#ifdef CONFIG_ERRATA_ANDES
46	case ANDES_VENDOR_ID:
47		cpu_mfr_info->patch_func = andes_errata_patch_func;
48		break;
49#endif
50#ifdef CONFIG_ERRATA_SIFIVE
51	case SIFIVE_VENDOR_ID:
52		cpu_mfr_info->patch_func = sifive_errata_patch_func;
53		break;
54#endif
55#ifdef CONFIG_ERRATA_THEAD
56	case THEAD_VENDOR_ID:
57		cpu_mfr_info->patch_func = thead_errata_patch_func;
58		break;
59#endif
60	default:
61		cpu_mfr_info->patch_func = NULL;
62	}
63}
64
65static u32 riscv_instruction_at(void *p)
66{
67	u16 *parcel = p;
68
69	return (u32)parcel[0] | (u32)parcel[1] << 16;
70}
71
72static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn,
73					     u32 jalr_insn, int patch_offset)
74{
75	u32 call[2] = { auipc_insn, jalr_insn };
76	s32 imm;
77
78	/* get and adjust new target address */
79	imm = riscv_insn_extract_utype_itype_imm(auipc_insn, jalr_insn);
80	imm -= patch_offset;
81
82	/* update instructions */
83	riscv_insn_insert_utype_itype_imm(&call[0], &call[1], imm);
84
85	/* patch the call place again */
86	patch_text_nosync(ptr, call, sizeof(u32) * 2);
87}
88
89static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset)
90{
91	s32 imm;
92
93	/* get and adjust new target address */
94	imm = riscv_insn_extract_jtype_imm(jal_insn);
95	imm -= patch_offset;
96
97	/* update instruction */
98	riscv_insn_insert_jtype_imm(&jal_insn, imm);
99
100	/* patch the call place again */
101	patch_text_nosync(ptr, &jal_insn, sizeof(u32));
102}
103
104void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,
105				      int patch_offset)
106{
107	int num_insn = len / sizeof(u32);
108	int i;
109
110	for (i = 0; i < num_insn; i++) {
111		u32 insn = riscv_instruction_at(alt_ptr + i * sizeof(u32));
112
113		/*
114		 * May be the start of an auipc + jalr pair
115		 * Needs to check that at least one more instruction
116		 * is in the list.
117		 */
118		if (riscv_insn_is_auipc(insn) && i < num_insn - 1) {
119			u32 insn2 = riscv_instruction_at(alt_ptr + (i + 1) * sizeof(u32));
120
121			if (!riscv_insn_is_jalr(insn2))
122				continue;
123
124			/* if instruction pair is a call, it will use the ra register */
125			if (RV_EXTRACT_RD_REG(insn) != 1)
126				continue;
127
128			riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32),
129							 insn, insn2, patch_offset);
130			i++;
131		}
132
133		if (riscv_insn_is_jal(insn)) {
134			s32 imm = riscv_insn_extract_jtype_imm(insn);
135
136			/* Don't modify jumps inside the alternative block */
137			if ((alt_ptr + i * sizeof(u32) + imm) >= alt_ptr &&
138			    (alt_ptr + i * sizeof(u32) + imm) < (alt_ptr + len))
139				continue;
140
141			riscv_alternative_fix_jal(alt_ptr + i * sizeof(u32),
142						  insn, patch_offset);
143		}
144	}
145}
146
147/*
148 * This is called very early in the boot process (directly after we run
149 * a feature detect on the boot CPU). No need to worry about other CPUs
150 * here.
151 */
152static void __init_or_module _apply_alternatives(struct alt_entry *begin,
153						 struct alt_entry *end,
154						 unsigned int stage)
155{
156	struct cpu_manufacturer_info_t cpu_mfr_info;
157
158	riscv_fill_cpu_mfr_info(&cpu_mfr_info);
159
160	riscv_cpufeature_patch_func(begin, end, stage);
161
162	if (!cpu_mfr_info.patch_func)
163		return;
164
165	cpu_mfr_info.patch_func(begin, end,
166				cpu_mfr_info.arch_id,
167				cpu_mfr_info.imp_id,
168				stage);
169}
170
171#ifdef CONFIG_MMU
172static void __init apply_vdso_alternatives(void)
173{
174	const Elf_Ehdr *hdr;
175	const Elf_Shdr *shdr;
176	const Elf_Shdr *alt;
177	struct alt_entry *begin, *end;
178
179	hdr = (Elf_Ehdr *)vdso_start;
180	shdr = (void *)hdr + hdr->e_shoff;
181	alt = find_section(hdr, shdr, ".alternative");
182	if (!alt)
183		return;
184
185	begin = (void *)hdr + alt->sh_offset,
186	end = (void *)hdr + alt->sh_offset + alt->sh_size,
187
188	_apply_alternatives((struct alt_entry *)begin,
189			    (struct alt_entry *)end,
190			    RISCV_ALTERNATIVES_BOOT);
191}
192#else
193static void __init apply_vdso_alternatives(void) { }
194#endif
195
196void __init apply_boot_alternatives(void)
197{
198	/* If called on non-boot cpu things could go wrong */
199	WARN_ON(smp_processor_id() != 0);
200
201	_apply_alternatives((struct alt_entry *)__alt_start,
202			    (struct alt_entry *)__alt_end,
203			    RISCV_ALTERNATIVES_BOOT);
204
205	apply_vdso_alternatives();
206}
207
208/*
209 * apply_early_boot_alternatives() is called from setup_vm() with MMU-off.
210 *
211 * Following requirements should be honoured for it to work correctly:
212 * 1) It should use PC-relative addressing for accessing kernel symbols.
213 *    To achieve this we always use GCC cmodel=medany.
214 * 2) The compiler instrumentation for FTRACE will not work for setup_vm()
215 *    so disable compiler instrumentation when FTRACE is enabled.
216 *
217 * Currently, the above requirements are honoured by using custom CFLAGS
218 * for alternative.o in kernel/Makefile.
219 */
220void __init apply_early_boot_alternatives(void)
221{
222#ifdef CONFIG_RISCV_ALTERNATIVE_EARLY
223	_apply_alternatives((struct alt_entry *)__alt_start,
224			    (struct alt_entry *)__alt_end,
225			    RISCV_ALTERNATIVES_EARLY_BOOT);
226#endif
227}
228
229#ifdef CONFIG_MODULES
230void apply_module_alternatives(void *start, size_t length)
231{
232	_apply_alternatives((struct alt_entry *)start,
233			    (struct alt_entry *)(start + length),
234			    RISCV_ALTERNATIVES_MODULE);
235}
236#endif
237