1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * alternative runtime patching 4 * inspired by the ARM64 and x86 version 5 * 6 * Copyright (C) 2021 Sifive. 7 */ 8 9#include <linux/init.h> 10#include <linux/module.h> 11#include <linux/cpu.h> 12#include <linux/uaccess.h> 13#include <asm/alternative.h> 14#include <asm/module.h> 15#include <asm/sections.h> 16#include <asm/vdso.h> 17#include <asm/vendorid_list.h> 18#include <asm/sbi.h> 19#include <asm/csr.h> 20#include <asm/insn.h> 21#include <asm/patch.h> 22 23struct cpu_manufacturer_info_t { 24 unsigned long vendor_id; 25 unsigned long arch_id; 26 unsigned long imp_id; 27 void (*patch_func)(struct alt_entry *begin, struct alt_entry *end, 28 unsigned long archid, unsigned long impid, 29 unsigned int stage); 30}; 31 32static void riscv_fill_cpu_mfr_info(struct cpu_manufacturer_info_t *cpu_mfr_info) 33{ 34#ifdef CONFIG_RISCV_M_MODE 35 cpu_mfr_info->vendor_id = csr_read(CSR_MVENDORID); 36 cpu_mfr_info->arch_id = csr_read(CSR_MARCHID); 37 cpu_mfr_info->imp_id = csr_read(CSR_MIMPID); 38#else 39 cpu_mfr_info->vendor_id = sbi_get_mvendorid(); 40 cpu_mfr_info->arch_id = sbi_get_marchid(); 41 cpu_mfr_info->imp_id = sbi_get_mimpid(); 42#endif 43 44 switch (cpu_mfr_info->vendor_id) { 45#ifdef CONFIG_ERRATA_ANDES 46 case ANDES_VENDOR_ID: 47 cpu_mfr_info->patch_func = andes_errata_patch_func; 48 break; 49#endif 50#ifdef CONFIG_ERRATA_SIFIVE 51 case SIFIVE_VENDOR_ID: 52 cpu_mfr_info->patch_func = sifive_errata_patch_func; 53 break; 54#endif 55#ifdef CONFIG_ERRATA_THEAD 56 case THEAD_VENDOR_ID: 57 cpu_mfr_info->patch_func = thead_errata_patch_func; 58 break; 59#endif 60 default: 61 cpu_mfr_info->patch_func = NULL; 62 } 63} 64 65static u32 riscv_instruction_at(void *p) 66{ 67 u16 *parcel = p; 68 69 return (u32)parcel[0] | (u32)parcel[1] << 16; 70} 71 72static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn, 73 u32 jalr_insn, int patch_offset) 74{ 75 u32 call[2] = { auipc_insn, jalr_insn }; 76 s32 imm; 77 78 /* get and adjust new target address */ 79 imm = riscv_insn_extract_utype_itype_imm(auipc_insn, jalr_insn); 80 imm -= patch_offset; 81 82 /* update instructions */ 83 riscv_insn_insert_utype_itype_imm(&call[0], &call[1], imm); 84 85 /* patch the call place again */ 86 patch_text_nosync(ptr, call, sizeof(u32) * 2); 87} 88 89static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset) 90{ 91 s32 imm; 92 93 /* get and adjust new target address */ 94 imm = riscv_insn_extract_jtype_imm(jal_insn); 95 imm -= patch_offset; 96 97 /* update instruction */ 98 riscv_insn_insert_jtype_imm(&jal_insn, imm); 99 100 /* patch the call place again */ 101 patch_text_nosync(ptr, &jal_insn, sizeof(u32)); 102} 103 104void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, 105 int patch_offset) 106{ 107 int num_insn = len / sizeof(u32); 108 int i; 109 110 for (i = 0; i < num_insn; i++) { 111 u32 insn = riscv_instruction_at(alt_ptr + i * sizeof(u32)); 112 113 /* 114 * May be the start of an auipc + jalr pair 115 * Needs to check that at least one more instruction 116 * is in the list. 117 */ 118 if (riscv_insn_is_auipc(insn) && i < num_insn - 1) { 119 u32 insn2 = riscv_instruction_at(alt_ptr + (i + 1) * sizeof(u32)); 120 121 if (!riscv_insn_is_jalr(insn2)) 122 continue; 123 124 /* if instruction pair is a call, it will use the ra register */ 125 if (RV_EXTRACT_RD_REG(insn) != 1) 126 continue; 127 128 riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32), 129 insn, insn2, patch_offset); 130 i++; 131 } 132 133 if (riscv_insn_is_jal(insn)) { 134 s32 imm = riscv_insn_extract_jtype_imm(insn); 135 136 /* Don't modify jumps inside the alternative block */ 137 if ((alt_ptr + i * sizeof(u32) + imm) >= alt_ptr && 138 (alt_ptr + i * sizeof(u32) + imm) < (alt_ptr + len)) 139 continue; 140 141 riscv_alternative_fix_jal(alt_ptr + i * sizeof(u32), 142 insn, patch_offset); 143 } 144 } 145} 146 147/* 148 * This is called very early in the boot process (directly after we run 149 * a feature detect on the boot CPU). No need to worry about other CPUs 150 * here. 151 */ 152static void __init_or_module _apply_alternatives(struct alt_entry *begin, 153 struct alt_entry *end, 154 unsigned int stage) 155{ 156 struct cpu_manufacturer_info_t cpu_mfr_info; 157 158 riscv_fill_cpu_mfr_info(&cpu_mfr_info); 159 160 riscv_cpufeature_patch_func(begin, end, stage); 161 162 if (!cpu_mfr_info.patch_func) 163 return; 164 165 cpu_mfr_info.patch_func(begin, end, 166 cpu_mfr_info.arch_id, 167 cpu_mfr_info.imp_id, 168 stage); 169} 170 171#ifdef CONFIG_MMU 172static void __init apply_vdso_alternatives(void) 173{ 174 const Elf_Ehdr *hdr; 175 const Elf_Shdr *shdr; 176 const Elf_Shdr *alt; 177 struct alt_entry *begin, *end; 178 179 hdr = (Elf_Ehdr *)vdso_start; 180 shdr = (void *)hdr + hdr->e_shoff; 181 alt = find_section(hdr, shdr, ".alternative"); 182 if (!alt) 183 return; 184 185 begin = (void *)hdr + alt->sh_offset, 186 end = (void *)hdr + alt->sh_offset + alt->sh_size, 187 188 _apply_alternatives((struct alt_entry *)begin, 189 (struct alt_entry *)end, 190 RISCV_ALTERNATIVES_BOOT); 191} 192#else 193static void __init apply_vdso_alternatives(void) { } 194#endif 195 196void __init apply_boot_alternatives(void) 197{ 198 /* If called on non-boot cpu things could go wrong */ 199 WARN_ON(smp_processor_id() != 0); 200 201 _apply_alternatives((struct alt_entry *)__alt_start, 202 (struct alt_entry *)__alt_end, 203 RISCV_ALTERNATIVES_BOOT); 204 205 apply_vdso_alternatives(); 206} 207 208/* 209 * apply_early_boot_alternatives() is called from setup_vm() with MMU-off. 210 * 211 * Following requirements should be honoured for it to work correctly: 212 * 1) It should use PC-relative addressing for accessing kernel symbols. 213 * To achieve this we always use GCC cmodel=medany. 214 * 2) The compiler instrumentation for FTRACE will not work for setup_vm() 215 * so disable compiler instrumentation when FTRACE is enabled. 216 * 217 * Currently, the above requirements are honoured by using custom CFLAGS 218 * for alternative.o in kernel/Makefile. 219 */ 220void __init apply_early_boot_alternatives(void) 221{ 222#ifdef CONFIG_RISCV_ALTERNATIVE_EARLY 223 _apply_alternatives((struct alt_entry *)__alt_start, 224 (struct alt_entry *)__alt_end, 225 RISCV_ALTERNATIVES_EARLY_BOOT); 226#endif 227} 228 229#ifdef CONFIG_MODULES 230void apply_module_alternatives(void *start, size_t length) 231{ 232 _apply_alternatives((struct alt_entry *)start, 233 (struct alt_entry *)(start + length), 234 RISCV_ALTERNATIVES_MODULE); 235} 236#endif 237