1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 *
22 * $FreeBSD$
23 *
24 */
25/*
26 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
27 * Use is subject to license terms.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/types.h>
36#include <sys/kernel.h>
37#include <sys/malloc.h>
38#include <sys/kmem.h>
39#include <sys/smp.h>
40#include <sys/dtrace_impl.h>
41#include <sys/dtrace_bsd.h>
42#include <machine/armreg.h>
43#include <machine/clock.h>
44#include <machine/frame.h>
45#include <machine/trap.h>
46#include <machine/vmparam.h>
47#include <vm/pmap.h>
48
49extern dtrace_id_t	dtrace_probeid_error;
50extern int (*dtrace_invop_jump_addr)(struct trapframe *);
51extern void dtrace_getnanotime(struct timespec *tsp);
52extern void dtrace_getnanouptime(struct timespec *tsp);
53
54int dtrace_invop(uintptr_t, struct trapframe *, uintptr_t);
55void dtrace_invop_init(void);
56void dtrace_invop_uninit(void);
57
58typedef struct dtrace_invop_hdlr {
59	int (*dtih_func)(uintptr_t, struct trapframe *, uintptr_t);
60	struct dtrace_invop_hdlr *dtih_next;
61} dtrace_invop_hdlr_t;
62
63dtrace_invop_hdlr_t *dtrace_invop_hdlr;
64
65int
66dtrace_invop(uintptr_t addr, struct trapframe *frame, uintptr_t eax)
67{
68	dtrace_invop_hdlr_t *hdlr;
69	int rval;
70
71	for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next)
72		if ((rval = hdlr->dtih_func(addr, frame, eax)) != 0)
73			return (rval);
74
75	return (0);
76}
77
78
79void
80dtrace_invop_add(int (*func)(uintptr_t, struct trapframe *, uintptr_t))
81{
82	dtrace_invop_hdlr_t *hdlr;
83
84	hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP);
85	hdlr->dtih_func = func;
86	hdlr->dtih_next = dtrace_invop_hdlr;
87	dtrace_invop_hdlr = hdlr;
88}
89
90void
91dtrace_invop_remove(int (*func)(uintptr_t, struct trapframe *, uintptr_t))
92{
93	dtrace_invop_hdlr_t *hdlr, *prev;
94
95	hdlr = dtrace_invop_hdlr;
96	prev = NULL;
97
98	for (;;) {
99		if (hdlr == NULL)
100			panic("attempt to remove non-existent invop handler");
101
102		if (hdlr->dtih_func == func)
103			break;
104
105		prev = hdlr;
106		hdlr = hdlr->dtih_next;
107	}
108
109	if (prev == NULL) {
110		ASSERT(dtrace_invop_hdlr == hdlr);
111		dtrace_invop_hdlr = hdlr->dtih_next;
112	} else {
113		ASSERT(dtrace_invop_hdlr != hdlr);
114		prev->dtih_next = hdlr->dtih_next;
115	}
116
117	kmem_free(hdlr, 0);
118}
119
120/*ARGSUSED*/
121void
122dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
123{
124
125	(*func)(0, (uintptr_t)VM_MIN_KERNEL_ADDRESS);
126}
127
128void
129dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg)
130{
131	cpuset_t cpus;
132
133	if (cpu == DTRACE_CPUALL)
134		cpus = all_cpus;
135	else
136		CPU_SETOF(cpu, &cpus);
137
138	smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier, func,
139	    smp_no_rendezvous_barrier, arg);
140}
141
142static void
143dtrace_sync_func(void)
144{
145
146}
147
148void
149dtrace_sync(void)
150{
151
152	dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL);
153}
154
155/*
156 * DTrace needs a high resolution time function which can be called from a
157 * probe context and guaranteed not to have instrumented with probes itself.
158 *
159 * Returns nanoseconds since some arbitrary point in time (likely SoC reset?).
160 */
161uint64_t
162dtrace_gethrtime(void)
163{
164	uint64_t count, freq;
165
166	count = READ_SPECIALREG(cntvct_el0);
167	freq = READ_SPECIALREG(cntfrq_el0);
168	return ((1000000000UL * count) / freq);
169}
170
171/*
172 * Return a much lower resolution wallclock time based on the system clock
173 * updated by the timer.  If needed, we could add a version interpolated from
174 * the system clock as is the case with dtrace_gethrtime().
175 */
176uint64_t
177dtrace_gethrestime(void)
178{
179	struct timespec current_time;
180
181	dtrace_getnanotime(&current_time);
182
183	return (current_time.tv_sec * 1000000000UL + current_time.tv_nsec);
184}
185
186/* Function to handle DTrace traps during probes. See arm64/arm64/trap.c */
187int
188dtrace_trap(struct trapframe *frame, u_int type)
189{
190	/*
191	 * A trap can occur while DTrace executes a probe. Before
192	 * executing the probe, DTrace blocks re-scheduling and sets
193	 * a flag in its per-cpu flags to indicate that it doesn't
194	 * want to fault. On returning from the probe, the no-fault
195	 * flag is cleared and finally re-scheduling is enabled.
196	 *
197	 * Check if DTrace has enabled 'no-fault' mode:
198	 *
199	 */
200
201	if ((cpu_core[curcpu].cpuc_dtrace_flags & CPU_DTRACE_NOFAULT) != 0) {
202		/*
203		 * There are only a couple of trap types that are expected.
204		 * All the rest will be handled in the usual way.
205		 */
206		switch (type) {
207		case EXCP_DATA_ABORT:
208			/* Flag a bad address. */
209			cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR;
210			cpu_core[curcpu].cpuc_dtrace_illval = 0;
211
212			/*
213			 * Offset the instruction pointer to the instruction
214			 * following the one causing the fault.
215			 */
216			frame->tf_elr += 4;
217			return (1);
218		default:
219			/* Handle all other traps in the usual way. */
220			break;
221		}
222	}
223
224	/* Handle the trap in the usual way. */
225	return (0);
226}
227
228void
229dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
230    int fault, int fltoffs, uintptr_t illval)
231{
232
233	dtrace_probe(dtrace_probeid_error, (uint64_t)(uintptr_t)state,
234	    (uintptr_t)epid,
235	    (uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs);
236}
237
238static void
239dtrace_load64(uint64_t *addr, struct trapframe *frame, u_int reg)
240{
241
242	KASSERT(reg <= 31, ("dtrace_load64: Invalid register %u", reg));
243	if (reg < nitems(frame->tf_x))
244		frame->tf_x[reg] = *addr;
245	else if (reg == 30) /* lr */
246		frame->tf_lr = *addr;
247	/* Nothing to do for load to xzr */
248}
249
250static void
251dtrace_store64(uint64_t *addr, struct trapframe *frame, u_int reg)
252{
253
254	KASSERT(reg <= 31, ("dtrace_store64: Invalid register %u", reg));
255	if (reg < nitems(frame->tf_x))
256		*addr = frame->tf_x[reg];
257	else if (reg == 30) /* lr */
258		*addr = frame->tf_lr;
259	else if (reg == 31) /* xzr */
260		*addr = 0;
261}
262
263static int
264dtrace_invop_start(struct trapframe *frame)
265{
266	int data, invop, reg, update_sp;
267	register_t arg1, arg2;
268	register_t *sp;
269	int offs;
270	int tmp;
271	int i;
272
273	invop = dtrace_invop(frame->tf_elr, frame, frame->tf_elr);
274
275	tmp = (invop & LDP_STP_MASK);
276	if (tmp == STP_64 || tmp == LDP_64) {
277		sp = (register_t *)frame->tf_sp;
278		data = invop;
279		arg1 = (data >> ARG1_SHIFT) & ARG1_MASK;
280		arg2 = (data >> ARG2_SHIFT) & ARG2_MASK;
281
282		offs = (data >> OFFSET_SHIFT) & OFFSET_MASK;
283
284		switch (tmp) {
285		case STP_64:
286			if (offs >> (OFFSET_SIZE - 1))
287				sp -= (~offs & OFFSET_MASK) + 1;
288			else
289				sp += (offs);
290			dtrace_store64(sp + 0, frame, arg1);
291			dtrace_store64(sp + 1, frame, arg2);
292			break;
293		case LDP_64:
294			dtrace_load64(sp + 0, frame, arg1);
295			dtrace_load64(sp + 1, frame, arg2);
296			if (offs >> (OFFSET_SIZE - 1))
297				sp -= (~offs & OFFSET_MASK) + 1;
298			else
299				sp += (offs);
300			break;
301		default:
302			break;
303		}
304
305		/* Update the stack pointer and program counter to continue */
306		frame->tf_sp = (register_t)sp;
307		frame->tf_elr += INSN_SIZE;
308		return (0);
309	}
310
311	if ((invop & SUB_MASK) == SUB_INSTR) {
312		frame->tf_sp -= (invop >> SUB_IMM_SHIFT) & SUB_IMM_MASK;
313		frame->tf_elr += INSN_SIZE;
314		return (0);
315	}
316
317	if ((invop & B_MASK) == B_INSTR) {
318		data = (invop & B_DATA_MASK);
319		/* The data is the number of 4-byte words to change the pc */
320		data *= 4;
321		frame->tf_elr += data;
322		return (0);
323	}
324
325	if (invop == RET_INSTR) {
326		frame->tf_elr = frame->tf_lr;
327		return (0);
328	}
329
330	return (-1);
331}
332
333void
334dtrace_invop_init(void)
335{
336
337	dtrace_invop_jump_addr = dtrace_invop_start;
338}
339
340void
341dtrace_invop_uninit(void)
342{
343
344	dtrace_invop_jump_addr = 0;
345}
346