151078Speter// SPDX-License-Identifier: GPL-2.0-only 251078Speter/* 351078Speter * Copyright (C) 2020 SiFive 451078Speter */ 551078Speter 651078Speter#include <linux/spinlock.h> 751078Speter#include <linux/mm.h> 851078Speter#include <linux/memory.h> 951078Speter#include <linux/string.h> 1051078Speter#include <linux/uaccess.h> 1151078Speter#include <linux/stop_machine.h> 1251078Speter#include <asm/kprobes.h> 1351078Speter#include <asm/cacheflush.h> 1451078Speter#include <asm/fixmap.h> 1551078Speter#include <asm/ftrace.h> 1651078Speter#include <asm/patch.h> 1751078Speter#include <asm/sections.h> 1851078Speter 1951078Speterstruct patch_insn { 2051078Speter void *addr; 2151078Speter u32 *insns; 2251078Speter int ninsns; 2351078Speter atomic_t cpu_count; 2451078Speter}; 2551078Speter 2651078Speterint riscv_patch_in_stop_machine = false; 2751078Speter 2851078Speter#ifdef CONFIG_MMU 2951078Speter 3051078Speterstatic inline bool is_kernel_exittext(uintptr_t addr) 3151078Speter{ 3251078Speter return system_state < SYSTEM_RUNNING && 3351078Speter addr >= (uintptr_t)__exittext_begin && 3451078Speter addr < (uintptr_t)__exittext_end; 3551078Speter} 3651078Speter 37119419Sobrien/* 38119419Sobrien * The fix_to_virt(, idx) needs a const value (not a dynamic variable of 39119419Sobrien * reg-a0) or BUILD_BUG_ON failed with "idx >= __end_of_fixed_addresses". 4051078Speter * So use '__always_inline' and 'const unsigned int fixmap' here. 4151078Speter */ 4251078Speterstatic __always_inline void *patch_map(void *addr, const unsigned int fixmap) 4351078Speter{ 4451078Speter uintptr_t uintaddr = (uintptr_t) addr; 4551078Speter struct page *page; 4651078Speter 4751078Speter if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr)) 4851078Speter page = phys_to_page(__pa_symbol(addr)); 4951078Speter else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) 5051078Speter page = vmalloc_to_page(addr); 5151078Speter else 5251078Speter return addr; 5351078Speter 5451078Speter BUG_ON(!page); 5576166Smarkm 5665822Sjhb return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + 5751078Speter (uintaddr & ~PAGE_MASK)); 5851078Speter} 5951078Speter 6051078Speterstatic void patch_unmap(int fixmap) 61114216Skan{ 6276166Smarkm clear_fixmap(fixmap); 6376166Smarkm} 6476166SmarkmNOKPROBE_SYMBOL(patch_unmap); 6576166Smarkm 6676166Smarkmstatic int __patch_insn_set(void *addr, u8 c, size_t len) 6776166Smarkm{ 6876166Smarkm void *waddr = addr; 6951078Speter bool across_pages = (((uintptr_t)addr & ~PAGE_MASK) + len) > PAGE_SIZE; 7076166Smarkm 7160471Snyan /* 7251078Speter * Only two pages can be mapped at a time for writing. 7351078Speter */ 7451078Speter if (len + offset_in_page(addr) > 2 * PAGE_SIZE) 7593466Sbde return -EINVAL; 76119485Snjl /* 77119485Snjl * Before reaching here, it was expected to lock the text_mutex 78119485Snjl * already, so we don't need to give another lock here and could 79119485Snjl * ensure that it was safe between each cores. 8051078Speter */ 8186909Simp lockdep_assert_held(&text_mutex); 8286909Simp 8351078Speter preempt_disable(); 8451078Speter 8585302Simp if (across_pages) 8685365Simp patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1); 8751078Speter 8851078Speter waddr = patch_map(addr, FIX_TEXT_POKE0); 8977726Sjoerg 9051078Speter memset(waddr, c, len); 9177726Sjoerg 9251078Speter patch_unmap(FIX_TEXT_POKE0); 9351078Speter 9451078Speter if (across_pages) 9551078Speter patch_unmap(FIX_TEXT_POKE1); 9651078Speter 9751078Speter preempt_enable(); 9851078Speter 9951078Speter return 0; 10093470Sbde} 10193470SbdeNOKPROBE_SYMBOL(__patch_insn_set); 10293470Sbde 10393470Sbdestatic int __patch_insn_write(void *addr, const void *insn, size_t len) 10451078Speter{ 10551078Speter void *waddr = addr; 10651078Speter bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE; 10751078Speter int ret; 10851078Speter 10951078Speter /* 11051078Speter * Only two pages can be mapped at a time for writing. 11151078Speter */ 112104067Sphk if (len + offset_in_page(addr) > 2 * PAGE_SIZE) 113104067Sphk return -EINVAL; 11451078Speter 11551078Speter /* 116120175Sbde * Before reaching here, it was expected to lock the text_mutex 11751078Speter * already, so we don't need to give another lock here and could 118120175Sbde * ensure that it was safe between each cores. 119120175Sbde * 12051078Speter * We're currently using stop_machine() for ftrace & kprobes, and while 121120175Sbde * that ensures text_mutex is held before installing the mappings it 12251078Speter * does not ensure text_mutex is held by the calling thread. That's 12351078Speter * safe but triggers a lockdep failure, so just elide it for that 124120175Sbde * specific case. 125120175Sbde */ 126120175Sbde if (!riscv_patch_in_stop_machine) 127111613Sphk lockdep_assert_held(&text_mutex); 128120175Sbde 129112384Ssobomax preempt_disable(); 13051078Speter 13160471Snyan if (across_pages) 13260471Snyan patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1); 13360471Snyan 13460471Snyan waddr = patch_map(addr, FIX_TEXT_POKE0); 13560471Snyan 13651078Speter ret = copy_to_kernel_nofault(waddr, insn, len); 13751078Speter 13851078Speter patch_unmap(FIX_TEXT_POKE0); 13951078Speter 14051078Speter if (across_pages) 14151078Speter patch_unmap(FIX_TEXT_POKE1); 14251078Speter 14351078Speter preempt_enable(); 14453344Speter 14551078Speter return ret; 14651078Speter} 14751078SpeterNOKPROBE_SYMBOL(__patch_insn_write); 14851078Speter#else 14951078Speterstatic int __patch_insn_set(void *addr, u8 c, size_t len) 15051078Speter{ 15151078Speter memset(addr, c, len); 15251078Speter 15351078Speter return 0; 15451078Speter} 15551078SpeterNOKPROBE_SYMBOL(__patch_insn_set); 15651078Speter 15751078Speterstatic int __patch_insn_write(void *addr, const void *insn, size_t len) 15851078Speter{ 15951078Speter return copy_to_kernel_nofault(addr, insn, len); 16051078Speter} 16151078SpeterNOKPROBE_SYMBOL(__patch_insn_write); 16251078Speter#endif /* CONFIG_MMU */ 16351078Speter 16451078Speterstatic int patch_insn_set(void *addr, u8 c, size_t len) 16551078Speter{ 16651078Speter size_t patched = 0; 16751078Speter size_t size; 16851078Speter int ret = 0; 16951078Speter 17051078Speter /* 17186909Simp * __patch_insn_set() can only work on 2 pages at a time so call it in a 17251078Speter * loop with len <= 2 * PAGE_SIZE. 17351078Speter */ 17486909Simp while (patched < len && !ret) { 17586909Simp size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched); 17686909Simp ret = __patch_insn_set(addr + patched, c, size); 17786909Simp 17886909Simp patched += size; 17986909Simp } 18086909Simp 18186909Simp return ret; 18286909Simp} 18386909SimpNOKPROBE_SYMBOL(patch_insn_set); 18486909Simp 18586909Simpint patch_text_set_nosync(void *addr, u8 c, size_t len) 18686909Simp{ 18786909Simp u32 *tp = addr; 18886909Simp int ret; 18986909Simp 19086909Simp ret = patch_insn_set(tp, c, len); 19151078Speter 19286909Simp if (!ret) 19386909Simp flush_icache_range((uintptr_t)tp, (uintptr_t)tp + len); 19486909Simp 19586909Simp return ret; 19686909Simp} 19786909SimpNOKPROBE_SYMBOL(patch_text_set_nosync); 19886909Simp 19986909Simpstatic int patch_insn_write(void *addr, const void *insn, size_t len) 20086909Simp{ 20186909Simp size_t patched = 0; 20286909Simp size_t size; 20386909Simp int ret = 0; 20486909Simp 20586909Simp /* 206120175Sbde * Copy the instructions to the destination address, two pages at a time 20786909Simp * because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE. 20886909Simp */ 209120189Sbde while (patched < len && !ret) { 21086909Simp size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched); 21186909Simp ret = __patch_insn_write(addr + patched, insn + patched, size); 21286909Simp 21386909Simp patched += size; 21486909Simp } 21586909Simp 21686909Simp return ret; 21786909Simp} 21886909SimpNOKPROBE_SYMBOL(patch_insn_write); 21986909Simp 22086909Simpint patch_text_nosync(void *addr, const void *insns, size_t len) 22186909Simp{ 22286909Simp u32 *tp = addr; 22386909Simp int ret; 22486909Simp 22586909Simp ret = patch_insn_write(tp, insns, len); 22686909Simp 22786909Simp if (!ret) 22886909Simp flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len); 22986909Simp 23086909Simp return ret; 23186909Simp} 23286909SimpNOKPROBE_SYMBOL(patch_text_nosync); 23386909Simp 23486909Simpstatic int patch_text_cb(void *data) 23586909Simp{ 23686909Simp struct patch_insn *patch = data; 23786909Simp unsigned long len; 23886909Simp int i, ret = 0; 23986909Simp 24086909Simp if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) { 241120189Sbde for (i = 0; ret == 0 && i < patch->ninsns; i++) { 24286909Simp len = GET_INSN_LENGTH(patch->insns[i]); 24386909Simp ret = patch_text_nosync(patch->addr + i * len, 24486909Simp &patch->insns[i], len); 24586909Simp } 24686909Simp atomic_inc(&patch->cpu_count); 24786909Simp } else { 24886909Simp while (atomic_read(&patch->cpu_count) <= num_online_cpus()) 24986909Simp cpu_relax(); 25086909Simp smp_mb(); 25186909Simp } 25286909Simp 25386909Simp return ret; 25486909Simp} 25586909SimpNOKPROBE_SYMBOL(patch_text_cb); 25686909Simp 25786909Simpint patch_text(void *addr, u32 *insns, int ninsns) 25886909Simp{ 25986909Simp int ret; 26086909Simp struct patch_insn patch = { 26186909Simp .addr = addr, 262111613Sphk .insns = insns, 263119485Snjl .ninsns = ninsns, 264119485Snjl .cpu_count = ATOMIC_INIT(0), 265119485Snjl }; 26686909Simp 26786909Simp /* 26886909Simp * kprobes takes text_mutex, before calling patch_text(), but as we call 26986909Simp * calls stop_machine(), the lockdep assertion in patch_insn_write() 27086909Simp * gets confused by the context in which the lock is taken. 27186909Simp * Instead, ensure the lock is held before calling stop_machine(), and 27289986Sjhay * set riscv_patch_in_stop_machine to skip the check in 27389986Sjhay * patch_insn_write(). 27486909Simp */ 27586909Simp lockdep_assert_held(&text_mutex); 276116120Sscottl riscv_patch_in_stop_machine = true; 277116120Sscottl ret = stop_machine_cpuslocked(patch_text_cb, &patch, cpu_online_mask); 278116120Sscottl riscv_patch_in_stop_machine = false; 27986909Simp return ret; 28086909Simp} 28186909SimpNOKPROBE_SYMBOL(patch_text); 28286909Simp