1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 *
5 * Derived from MIPS:
6 * Copyright (C) 1995 - 2000 by Ralf Baechle
7 */
8#include <linux/context_tracking.h>
9#include <linux/signal.h>
10#include <linux/sched.h>
11#include <linux/interrupt.h>
12#include <linux/kernel.h>
13#include <linux/entry-common.h>
14#include <linux/errno.h>
15#include <linux/string.h>
16#include <linux/types.h>
17#include <linux/ptrace.h>
18#include <linux/ratelimit.h>
19#include <linux/mman.h>
20#include <linux/mm.h>
21#include <linux/smp.h>
22#include <linux/kdebug.h>
23#include <linux/perf_event.h>
24#include <linux/uaccess.h>
25#include <linux/kfence.h>
26
27#include <asm/branch.h>
28#include <asm/exception.h>
29#include <asm/mmu_context.h>
30#include <asm/ptrace.h>
31
32int show_unhandled_signals = 1;
33
34static void __kprobes no_context(struct pt_regs *regs,
35			unsigned long write, unsigned long address)
36{
37	const int field = sizeof(unsigned long) * 2;
38
39	/* Are we prepared to handle this kernel fault?	 */
40	if (fixup_exception(regs))
41		return;
42
43	if (kfence_handle_page_fault(address, write, regs))
44		return;
45
46	/*
47	 * Oops. The kernel tried to access some bad page. We'll have to
48	 * terminate things with extreme prejudice.
49	 */
50	bust_spinlocks(1);
51
52	pr_alert("CPU %d Unable to handle kernel paging request at "
53	       "virtual address %0*lx, era == %0*lx, ra == %0*lx\n",
54	       raw_smp_processor_id(), field, address, field, regs->csr_era,
55	       field,  regs->regs[1]);
56	die("Oops", regs);
57}
58
59static void __kprobes do_out_of_memory(struct pt_regs *regs,
60			unsigned long write, unsigned long address)
61{
62	/*
63	 * We ran out of memory, call the OOM killer, and return the userspace
64	 * (which will retry the fault, or kill us if we got oom-killed).
65	 */
66	if (!user_mode(regs)) {
67		no_context(regs, write, address);
68		return;
69	}
70	pagefault_out_of_memory();
71}
72
73static void __kprobes do_sigbus(struct pt_regs *regs,
74		unsigned long write, unsigned long address, int si_code)
75{
76	/* Kernel mode? Handle exceptions or die */
77	if (!user_mode(regs)) {
78		no_context(regs, write, address);
79		return;
80	}
81
82	/*
83	 * Send a sigbus, regardless of whether we were in kernel
84	 * or user mode.
85	 */
86	current->thread.csr_badvaddr = address;
87	current->thread.trap_nr = read_csr_excode();
88	force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
89}
90
91static void __kprobes do_sigsegv(struct pt_regs *regs,
92		unsigned long write, unsigned long address, int si_code)
93{
94	const int field = sizeof(unsigned long) * 2;
95	static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10);
96
97	/* Kernel mode? Handle exceptions or die */
98	if (!user_mode(regs)) {
99		no_context(regs, write, address);
100		return;
101	}
102
103	/* User mode accesses just cause a SIGSEGV */
104	current->thread.csr_badvaddr = address;
105	if (!write)
106		current->thread.error_code = 1;
107	else
108		current->thread.error_code = 2;
109	current->thread.trap_nr = read_csr_excode();
110
111	if (show_unhandled_signals &&
112	    unhandled_signal(current, SIGSEGV) && __ratelimit(&ratelimit_state)) {
113		pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n",
114			current->comm,
115			write ? "write access to" : "read access from",
116			field, address);
117		pr_info("era = %0*lx in", field,
118			(unsigned long) regs->csr_era);
119		print_vma_addr(KERN_CONT " ", regs->csr_era);
120		pr_cont("\n");
121		pr_info("ra  = %0*lx in", field,
122			(unsigned long) regs->regs[1]);
123		print_vma_addr(KERN_CONT " ", regs->regs[1]);
124		pr_cont("\n");
125	}
126	force_sig_fault(SIGSEGV, si_code, (void __user *)address);
127}
128
129/*
130 * This routine handles page faults.  It determines the address,
131 * and the problem, and then passes it off to one of the appropriate
132 * routines.
133 */
134static void __kprobes __do_page_fault(struct pt_regs *regs,
135			unsigned long write, unsigned long address)
136{
137	int si_code = SEGV_MAPERR;
138	unsigned int flags = FAULT_FLAG_DEFAULT;
139	struct task_struct *tsk = current;
140	struct mm_struct *mm = tsk->mm;
141	struct vm_area_struct *vma = NULL;
142	vm_fault_t fault;
143
144	if (kprobe_page_fault(regs, current->thread.trap_nr))
145		return;
146
147	/*
148	 * We fault-in kernel-space virtual memory on-demand. The
149	 * 'reference' page table is init_mm.pgd.
150	 *
151	 * NOTE! We MUST NOT take any locks for this case. We may
152	 * be in an interrupt or a critical region, and should
153	 * only copy the information from the master page table,
154	 * nothing more.
155	 */
156	if (address & __UA_LIMIT) {
157		if (!user_mode(regs))
158			no_context(regs, write, address);
159		else
160			do_sigsegv(regs, write, address, si_code);
161		return;
162	}
163
164	/*
165	 * If we're in an interrupt or have no user
166	 * context, we must not take the fault..
167	 */
168	if (faulthandler_disabled() || !mm) {
169		do_sigsegv(regs, write, address, si_code);
170		return;
171	}
172
173	if (user_mode(regs))
174		flags |= FAULT_FLAG_USER;
175
176	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
177retry:
178	vma = lock_mm_and_find_vma(mm, address, regs);
179	if (unlikely(!vma))
180		goto bad_area_nosemaphore;
181	goto good_area;
182
183/*
184 * Something tried to access memory that isn't in our memory map..
185 * Fix it, but check if it's kernel or user first..
186 */
187bad_area:
188	mmap_read_unlock(mm);
189bad_area_nosemaphore:
190	do_sigsegv(regs, write, address, si_code);
191	return;
192
193/*
194 * Ok, we have a good vm_area for this memory access, so
195 * we can handle it..
196 */
197good_area:
198	si_code = SEGV_ACCERR;
199
200	if (write) {
201		flags |= FAULT_FLAG_WRITE;
202		if (!(vma->vm_flags & VM_WRITE))
203			goto bad_area;
204	} else {
205		if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs))
206			goto bad_area;
207		if (!(vma->vm_flags & (VM_READ | VM_WRITE)) && address != exception_era(regs))
208			goto bad_area;
209	}
210
211	/*
212	 * If for any reason at all we couldn't handle the fault,
213	 * make sure we exit gracefully rather than endlessly redo
214	 * the fault.
215	 */
216	fault = handle_mm_fault(vma, address, flags, regs);
217
218	if (fault_signal_pending(fault, regs)) {
219		if (!user_mode(regs))
220			no_context(regs, write, address);
221		return;
222	}
223
224	/* The fault is fully completed (including releasing mmap lock) */
225	if (fault & VM_FAULT_COMPLETED)
226		return;
227
228	if (unlikely(fault & VM_FAULT_RETRY)) {
229		flags |= FAULT_FLAG_TRIED;
230
231		/*
232		 * No need to mmap_read_unlock(mm) as we would
233		 * have already released it in __lock_page_or_retry
234		 * in mm/filemap.c.
235		 */
236		goto retry;
237	}
238	if (unlikely(fault & VM_FAULT_ERROR)) {
239		mmap_read_unlock(mm);
240		if (fault & VM_FAULT_OOM) {
241			do_out_of_memory(regs, write, address);
242			return;
243		} else if (fault & VM_FAULT_SIGSEGV) {
244			do_sigsegv(regs, write, address, si_code);
245			return;
246		} else if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
247			do_sigbus(regs, write, address, si_code);
248			return;
249		}
250		BUG();
251	}
252
253	mmap_read_unlock(mm);
254}
255
256asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
257			unsigned long write, unsigned long address)
258{
259	irqentry_state_t state = irqentry_enter(regs);
260
261	/* Enable interrupt if enabled in parent context */
262	if (likely(regs->csr_prmd & CSR_PRMD_PIE))
263		local_irq_enable();
264
265	__do_page_fault(regs, write, address);
266
267	local_irq_disable();
268
269	irqentry_exit(regs, state);
270}
271