1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2022-2023 Loongson Technology Corporation Limited
4 */
5#define pr_fmt(fmt) "hw-breakpoint: " fmt
6
7#include <linux/hw_breakpoint.h>
8#include <linux/kprobes.h>
9#include <linux/perf_event.h>
10
11#include <asm/hw_breakpoint.h>
12
13/* Breakpoint currently in use for each BRP. */
14static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[LOONGARCH_MAX_BRP]);
15
16/* Watchpoint currently in use for each WRP. */
17static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[LOONGARCH_MAX_WRP]);
18
19int hw_breakpoint_slots(int type)
20{
21	/*
22	 * We can be called early, so don't rely on
23	 * our static variables being initialised.
24	 */
25	switch (type) {
26	case TYPE_INST:
27		return get_num_brps();
28	case TYPE_DATA:
29		return get_num_wrps();
30	default:
31		pr_warn("unknown slot type: %d\n", type);
32		return 0;
33	}
34}
35
36#define READ_WB_REG_CASE(OFF, N, REG, T, VAL)		\
37	case (OFF + N):					\
38		LOONGARCH_CSR_WATCH_READ(N, REG, T, VAL);	\
39		break
40
41#define WRITE_WB_REG_CASE(OFF, N, REG, T, VAL)		\
42	case (OFF + N):					\
43		LOONGARCH_CSR_WATCH_WRITE(N, REG, T, VAL);	\
44		break
45
46#define GEN_READ_WB_REG_CASES(OFF, REG, T, VAL)		\
47	READ_WB_REG_CASE(OFF, 0, REG, T, VAL);		\
48	READ_WB_REG_CASE(OFF, 1, REG, T, VAL);		\
49	READ_WB_REG_CASE(OFF, 2, REG, T, VAL);		\
50	READ_WB_REG_CASE(OFF, 3, REG, T, VAL);		\
51	READ_WB_REG_CASE(OFF, 4, REG, T, VAL);		\
52	READ_WB_REG_CASE(OFF, 5, REG, T, VAL);		\
53	READ_WB_REG_CASE(OFF, 6, REG, T, VAL);		\
54	READ_WB_REG_CASE(OFF, 7, REG, T, VAL);
55
56#define GEN_WRITE_WB_REG_CASES(OFF, REG, T, VAL)	\
57	WRITE_WB_REG_CASE(OFF, 0, REG, T, VAL);		\
58	WRITE_WB_REG_CASE(OFF, 1, REG, T, VAL);		\
59	WRITE_WB_REG_CASE(OFF, 2, REG, T, VAL);		\
60	WRITE_WB_REG_CASE(OFF, 3, REG, T, VAL);		\
61	WRITE_WB_REG_CASE(OFF, 4, REG, T, VAL);		\
62	WRITE_WB_REG_CASE(OFF, 5, REG, T, VAL);		\
63	WRITE_WB_REG_CASE(OFF, 6, REG, T, VAL);		\
64	WRITE_WB_REG_CASE(OFF, 7, REG, T, VAL);
65
66static u64 read_wb_reg(int reg, int n, int t)
67{
68	u64 val = 0;
69
70	switch (reg + n) {
71	GEN_READ_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val);
72	GEN_READ_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val);
73	GEN_READ_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val);
74	GEN_READ_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val);
75	default:
76		pr_warn("Attempt to read from unknown breakpoint register %d\n", n);
77	}
78
79	return val;
80}
81NOKPROBE_SYMBOL(read_wb_reg);
82
83static void write_wb_reg(int reg, int n, int t, u64 val)
84{
85	switch (reg + n) {
86	GEN_WRITE_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val);
87	GEN_WRITE_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val);
88	GEN_WRITE_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val);
89	GEN_WRITE_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val);
90	default:
91		pr_warn("Attempt to write to unknown breakpoint register %d\n", n);
92	}
93}
94NOKPROBE_SYMBOL(write_wb_reg);
95
96enum hw_breakpoint_ops {
97	HW_BREAKPOINT_INSTALL,
98	HW_BREAKPOINT_UNINSTALL,
99};
100
101/*
102 * hw_breakpoint_slot_setup - Find and setup a perf slot according to operations
103 *
104 * @slots: pointer to array of slots
105 * @max_slots: max number of slots
106 * @bp: perf_event to setup
107 * @ops: operation to be carried out on the slot
108 *
109 * Return:
110 *	slot index on success
111 *	-ENOSPC if no slot is available/matches
112 *	-EINVAL on wrong operations parameter
113 */
114
115static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots,
116				    struct perf_event *bp, enum hw_breakpoint_ops ops)
117{
118	int i;
119	struct perf_event **slot;
120
121	for (i = 0; i < max_slots; ++i) {
122		slot = &slots[i];
123		switch (ops) {
124		case HW_BREAKPOINT_INSTALL:
125			if (!*slot) {
126				*slot = bp;
127				return i;
128			}
129			break;
130		case HW_BREAKPOINT_UNINSTALL:
131			if (*slot == bp) {
132				*slot = NULL;
133				return i;
134			}
135			break;
136		default:
137			pr_warn_once("Unhandled hw breakpoint ops %d\n", ops);
138			return -EINVAL;
139		}
140	}
141
142	return -ENOSPC;
143}
144
145void ptrace_hw_copy_thread(struct task_struct *tsk)
146{
147	memset(tsk->thread.hbp_break, 0, sizeof(tsk->thread.hbp_break));
148	memset(tsk->thread.hbp_watch, 0, sizeof(tsk->thread.hbp_watch));
149}
150
151/*
152 * Unregister breakpoints from this task and reset the pointers in the thread_struct.
153 */
154void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
155{
156	int i;
157	struct thread_struct *t = &tsk->thread;
158
159	for (i = 0; i < LOONGARCH_MAX_BRP; i++) {
160		if (t->hbp_break[i]) {
161			unregister_hw_breakpoint(t->hbp_break[i]);
162			t->hbp_break[i] = NULL;
163		}
164	}
165
166	for (i = 0; i < LOONGARCH_MAX_WRP; i++) {
167		if (t->hbp_watch[i]) {
168			unregister_hw_breakpoint(t->hbp_watch[i]);
169			t->hbp_watch[i] = NULL;
170		}
171	}
172}
173
174static int hw_breakpoint_control(struct perf_event *bp,
175				 enum hw_breakpoint_ops ops)
176{
177	u32 ctrl;
178	int i, max_slots, enable;
179	struct perf_event **slots;
180	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
181
182	if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) {
183		/* Breakpoint */
184		slots = this_cpu_ptr(bp_on_reg);
185		max_slots = boot_cpu_data.watch_ireg_count;
186	} else {
187		/* Watchpoint */
188		slots = this_cpu_ptr(wp_on_reg);
189		max_slots = boot_cpu_data.watch_dreg_count;
190	}
191
192	i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops);
193
194	if (WARN_ONCE(i < 0, "Can't find any breakpoint slot"))
195		return i;
196
197	switch (ops) {
198	case HW_BREAKPOINT_INSTALL:
199		/* Set the FWPnCFG/MWPnCFG 1~4 register. */
200		write_wb_reg(CSR_CFG_ADDR, i, 0, info->address);
201		write_wb_reg(CSR_CFG_ADDR, i, 1, info->address);
202		write_wb_reg(CSR_CFG_MASK, i, 0, info->mask);
203		write_wb_reg(CSR_CFG_MASK, i, 1, info->mask);
204		write_wb_reg(CSR_CFG_ASID, i, 0, 0);
205		write_wb_reg(CSR_CFG_ASID, i, 1, 0);
206		if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) {
207			write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE);
208		} else {
209			ctrl = encode_ctrl_reg(info->ctrl);
210			write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl | CTRL_PLV_ENABLE);
211		}
212		enable = csr_read64(LOONGARCH_CSR_CRMD);
213		csr_write64(CSR_CRMD_WE | enable, LOONGARCH_CSR_CRMD);
214		break;
215	case HW_BREAKPOINT_UNINSTALL:
216		/* Reset the FWPnCFG/MWPnCFG 1~4 register. */
217		write_wb_reg(CSR_CFG_ADDR, i, 0, 0);
218		write_wb_reg(CSR_CFG_ADDR, i, 1, 0);
219		write_wb_reg(CSR_CFG_MASK, i, 0, 0);
220		write_wb_reg(CSR_CFG_MASK, i, 1, 0);
221		write_wb_reg(CSR_CFG_CTRL, i, 0, 0);
222		write_wb_reg(CSR_CFG_CTRL, i, 1, 0);
223		write_wb_reg(CSR_CFG_ASID, i, 0, 0);
224		write_wb_reg(CSR_CFG_ASID, i, 1, 0);
225		break;
226	}
227
228	return 0;
229}
230
231/*
232 * Install a perf counter breakpoint.
233 */
234int arch_install_hw_breakpoint(struct perf_event *bp)
235{
236	return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL);
237}
238
239void arch_uninstall_hw_breakpoint(struct perf_event *bp)
240{
241	hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL);
242}
243
244static int get_hbp_len(u8 hbp_len)
245{
246	unsigned int len_in_bytes = 0;
247
248	switch (hbp_len) {
249	case LOONGARCH_BREAKPOINT_LEN_1:
250		len_in_bytes = 1;
251		break;
252	case LOONGARCH_BREAKPOINT_LEN_2:
253		len_in_bytes = 2;
254		break;
255	case LOONGARCH_BREAKPOINT_LEN_4:
256		len_in_bytes = 4;
257		break;
258	case LOONGARCH_BREAKPOINT_LEN_8:
259		len_in_bytes = 8;
260		break;
261	}
262
263	return len_in_bytes;
264}
265
266/*
267 * Check whether bp virtual address is in kernel space.
268 */
269int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
270{
271	unsigned int len;
272	unsigned long va;
273
274	va = hw->address;
275	len = get_hbp_len(hw->ctrl.len);
276
277	return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
278}
279
280/*
281 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
282 * Hopefully this will disappear when ptrace can bypass the conversion
283 * to generic breakpoint descriptions.
284 */
285int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
286			   int *gen_len, int *gen_type, int *offset)
287{
288	/* Type */
289	switch (ctrl.type) {
290	case LOONGARCH_BREAKPOINT_EXECUTE:
291		*gen_type = HW_BREAKPOINT_X;
292		break;
293	case LOONGARCH_BREAKPOINT_LOAD:
294		*gen_type = HW_BREAKPOINT_R;
295		break;
296	case LOONGARCH_BREAKPOINT_STORE:
297		*gen_type = HW_BREAKPOINT_W;
298		break;
299	case LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE:
300		*gen_type = HW_BREAKPOINT_RW;
301		break;
302	default:
303		return -EINVAL;
304	}
305
306	if (!ctrl.len)
307		return -EINVAL;
308
309	*offset = __ffs(ctrl.len);
310
311	/* Len */
312	switch (ctrl.len) {
313	case LOONGARCH_BREAKPOINT_LEN_1:
314		*gen_len = HW_BREAKPOINT_LEN_1;
315		break;
316	case LOONGARCH_BREAKPOINT_LEN_2:
317		*gen_len = HW_BREAKPOINT_LEN_2;
318		break;
319	case LOONGARCH_BREAKPOINT_LEN_4:
320		*gen_len = HW_BREAKPOINT_LEN_4;
321		break;
322	case LOONGARCH_BREAKPOINT_LEN_8:
323		*gen_len = HW_BREAKPOINT_LEN_8;
324		break;
325	default:
326		return -EINVAL;
327	}
328
329	return 0;
330}
331
332/*
333 * Construct an arch_hw_breakpoint from a perf_event.
334 */
335static int arch_build_bp_info(struct perf_event *bp,
336			      const struct perf_event_attr *attr,
337			      struct arch_hw_breakpoint *hw)
338{
339	/* Type */
340	switch (attr->bp_type) {
341	case HW_BREAKPOINT_X:
342		hw->ctrl.type = LOONGARCH_BREAKPOINT_EXECUTE;
343		break;
344	case HW_BREAKPOINT_R:
345		hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD;
346		break;
347	case HW_BREAKPOINT_W:
348		hw->ctrl.type = LOONGARCH_BREAKPOINT_STORE;
349		break;
350	case HW_BREAKPOINT_RW:
351		hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE;
352		break;
353	default:
354		return -EINVAL;
355	}
356
357	/* Len */
358	switch (attr->bp_len) {
359	case HW_BREAKPOINT_LEN_1:
360		hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_1;
361		break;
362	case HW_BREAKPOINT_LEN_2:
363		hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_2;
364		break;
365	case HW_BREAKPOINT_LEN_4:
366		hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_4;
367		break;
368	case HW_BREAKPOINT_LEN_8:
369		hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_8;
370		break;
371	default:
372		return -EINVAL;
373	}
374
375	/* Address */
376	hw->address = attr->bp_addr;
377
378	return 0;
379}
380
381/*
382 * Validate the arch-specific HW Breakpoint register settings.
383 */
384int hw_breakpoint_arch_parse(struct perf_event *bp,
385			     const struct perf_event_attr *attr,
386			     struct arch_hw_breakpoint *hw)
387{
388	int ret;
389	u64 alignment_mask, offset;
390
391	/* Build the arch_hw_breakpoint. */
392	ret = arch_build_bp_info(bp, attr, hw);
393	if (ret)
394		return ret;
395
396	if (hw->ctrl.type != LOONGARCH_BREAKPOINT_EXECUTE)
397		alignment_mask = 0x7;
398	else
399		alignment_mask = 0x3;
400	offset = hw->address & alignment_mask;
401
402	hw->address &= ~alignment_mask;
403	hw->ctrl.len <<= offset;
404
405	return 0;
406}
407
408static void update_bp_registers(struct pt_regs *regs, int enable, int type)
409{
410	u32 ctrl;
411	int i, max_slots;
412	struct perf_event **slots;
413	struct arch_hw_breakpoint *info;
414
415	switch (type) {
416	case 0:
417		slots = this_cpu_ptr(bp_on_reg);
418		max_slots = boot_cpu_data.watch_ireg_count;
419		break;
420	case 1:
421		slots = this_cpu_ptr(wp_on_reg);
422		max_slots = boot_cpu_data.watch_dreg_count;
423		break;
424	default:
425		return;
426	}
427
428	for (i = 0; i < max_slots; ++i) {
429		if (!slots[i])
430			continue;
431
432		info = counter_arch_bp(slots[i]);
433		if (enable) {
434			if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) {
435				write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE);
436				write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE);
437			} else {
438				ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1);
439				if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD)
440					ctrl |= 0x1 << MWPnCFG3_LoadEn;
441				if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE)
442					ctrl |= 0x1 << MWPnCFG3_StoreEn;
443				write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl);
444			}
445			regs->csr_prmd |= CSR_PRMD_PWE;
446		} else {
447			if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) {
448				write_wb_reg(CSR_CFG_CTRL, i, 0, 0);
449			} else {
450				ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1);
451				if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD)
452					ctrl &= ~0x1 << MWPnCFG3_LoadEn;
453				if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE)
454					ctrl &= ~0x1 << MWPnCFG3_StoreEn;
455				write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl);
456			}
457			regs->csr_prmd &= ~CSR_PRMD_PWE;
458		}
459	}
460}
461NOKPROBE_SYMBOL(update_bp_registers);
462
463/*
464 * Debug exception handlers.
465 */
466void breakpoint_handler(struct pt_regs *regs)
467{
468	int i;
469	struct perf_event *bp, **slots;
470
471	slots = this_cpu_ptr(bp_on_reg);
472
473	for (i = 0; i < boot_cpu_data.watch_ireg_count; ++i) {
474		bp = slots[i];
475		if (bp == NULL)
476			continue;
477		perf_bp_event(bp, regs);
478	}
479	update_bp_registers(regs, 0, 0);
480}
481NOKPROBE_SYMBOL(breakpoint_handler);
482
483void watchpoint_handler(struct pt_regs *regs)
484{
485	int i;
486	struct perf_event *wp, **slots;
487
488	slots = this_cpu_ptr(wp_on_reg);
489
490	for (i = 0; i < boot_cpu_data.watch_dreg_count; ++i) {
491		wp = slots[i];
492		if (wp == NULL)
493			continue;
494		perf_bp_event(wp, regs);
495	}
496	update_bp_registers(regs, 0, 1);
497}
498NOKPROBE_SYMBOL(watchpoint_handler);
499
500static int __init arch_hw_breakpoint_init(void)
501{
502	int cpu;
503
504	boot_cpu_data.watch_ireg_count = get_num_brps();
505	boot_cpu_data.watch_dreg_count = get_num_wrps();
506
507	pr_info("Found %d breakpoint and %d watchpoint registers.\n",
508		boot_cpu_data.watch_ireg_count, boot_cpu_data.watch_dreg_count);
509
510	for (cpu = 1; cpu < NR_CPUS; cpu++) {
511		cpu_data[cpu].watch_ireg_count = boot_cpu_data.watch_ireg_count;
512		cpu_data[cpu].watch_dreg_count = boot_cpu_data.watch_dreg_count;
513	}
514
515	return 0;
516}
517arch_initcall(arch_hw_breakpoint_init);
518
519void hw_breakpoint_thread_switch(struct task_struct *next)
520{
521	u64 addr, mask;
522	struct pt_regs *regs = task_pt_regs(next);
523
524	if (test_tsk_thread_flag(next, TIF_SINGLESTEP)) {
525		addr = read_wb_reg(CSR_CFG_ADDR, 0, 0);
526		mask = read_wb_reg(CSR_CFG_MASK, 0, 0);
527		if (!((regs->csr_era ^ addr) & ~mask))
528			csr_write32(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS);
529		regs->csr_prmd |= CSR_PRMD_PWE;
530	} else {
531		/* Update breakpoints */
532		update_bp_registers(regs, 1, 0);
533		/* Update watchpoints */
534		update_bp_registers(regs, 1, 1);
535	}
536}
537
538void hw_breakpoint_pmu_read(struct perf_event *bp)
539{
540}
541
542/*
543 * Dummy function to register with die_notifier.
544 */
545int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
546				    unsigned long val, void *data)
547{
548	return NOTIFY_DONE;
549}
550