1/*	$NetBSD: dtrace_asm.S,v 1.3.8.1 2012/11/22 17:39:28 riz Exp $	*/
2
3/*
4 * CDDL HEADER START
5 *
6 * The contents of this file are subject to the terms of the
7 * Common Development and Distribution License (the "License").
8 * You may not use this file except in compliance with the License.
9 *
10 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
11 * or http://www.opensolaris.org/os/licensing.
12 * See the License for the specific language governing permissions
13 * and limitations under the License.
14 *
15 * When distributing Covered Code, include this CDDL HEADER in each
16 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
17 * If applicable, add the following below this CDDL HEADER, with the
18 * fields enclosed by brackets "[]" replaced with your own identifying
19 * information: Portions Copyright [yyyy] [name of copyright owner]
20 *
21 * CDDL HEADER END
22 *
23 * Portions Copyright 2008 John Birrell <jb@freebsd.org>
24 *
25 * $FreeBSD: src/sys/cddl/dev/dtrace/amd64/dtrace_asm.S,v 1.1.4.1 2009/08/03 08:13:06 kensmith Exp $
26 *
27 */
28/*
29 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
30 * Use is subject to license terms.
31 */
32
33#define _ASM
34
35#include <sys/cpuvar_defs.h>
36#include <sys/dtrace.h>
37#include <machine/asm.h>
38#define END(a)
39#define MEXITCOUNT
40
41#include "assym.h"
42
43#define INTR_POP				\
44	MEXITCOUNT;				\
45	movq	TF_RDI(%rsp),%rdi;		\
46	movq	TF_RSI(%rsp),%rsi;		\
47	movq	TF_RDX(%rsp),%rdx;		\
48	movq	TF_RCX(%rsp),%rcx;		\
49	movq	TF_R8(%rsp),%r8;		\
50	movq	TF_R9(%rsp),%r9;		\
51	movq	TF_RAX(%rsp),%rax;		\
52	movq	TF_RBX(%rsp),%rbx;		\
53	movq	TF_RBP(%rsp),%rbp;		\
54	movq	TF_R10(%rsp),%r10;		\
55	movq	TF_R11(%rsp),%r11;		\
56	movq	TF_R12(%rsp),%r12;		\
57	movq	TF_R13(%rsp),%r13;		\
58	movq	TF_R14(%rsp),%r14;		\
59	movq	TF_R15(%rsp),%r15;		\
60	testb	$SEL_RPL_MASK,TF_CS(%rsp);	\
61	jz	1f;				\
62	cli;					\
63	swapgs;					\
641:	addq	$TF_RIP,%rsp;
65
66
67	ENTRY(dtrace_invop_start)
68
69	/*
70	 * #BP traps with %rip set to the next address. We need to decrement
71	 * the value to indicate the address of the int3 (0xcc) instruction
72	 * that we substituted.
73	 */
74	movq	TF_RIP(%rsp), %rdi
75	decq	%rdi
76	movq	TF_RSP(%rsp), %rsi
77	movq	TF_RAX(%rsp), %rdx
78	pushq	(%rsi)
79	movq	%rsp, %rsi
80	call	dtrace_invop
81	ALTENTRY(dtrace_invop_callsite)
82	addq	$8, %rsp
83	cmpl	$DTRACE_INVOP_PUSHL_EBP, %eax
84	je	bp_push
85	cmpl	$DTRACE_INVOP_LEAVE, %eax
86	je	bp_leave
87	cmpl	$DTRACE_INVOP_NOP, %eax
88	je	bp_nop
89	cmpl	$DTRACE_INVOP_RET, %eax
90	je	bp_ret
91
92	/* When all else fails handle the trap in the usual way. */
93	jmpq	*dtrace_invop_calltrap_addr
94
95bp_push:
96	/*
97	 * We must emulate a "pushq %rbp".  To do this, we pull the stack
98	 * down 8 bytes, and then store the base pointer.
99	 */
100	INTR_POP
101	subq	$16, %rsp		/* make room for %rbp */
102	pushq	%rax			/* push temp */
103	movq	24(%rsp), %rax		/* load calling RIP */
104	movq	%rax, 8(%rsp)		/* store calling RIP */
105	movq	32(%rsp), %rax		/* load calling CS */
106	movq	%rax, 16(%rsp)		/* store calling CS */
107	movq	40(%rsp), %rax		/* load calling RFLAGS */
108	movq	%rax, 24(%rsp)		/* store calling RFLAGS */
109	movq	48(%rsp), %rax		/* load calling RSP */
110	subq	$8, %rax		/* make room for %rbp */
111	movq	%rax, 32(%rsp)		/* store calling RSP */
112	movq	56(%rsp), %rax		/* load calling SS */
113	movq	%rax, 40(%rsp)		/* store calling SS */
114	movq	32(%rsp), %rax		/* reload calling RSP */
115	movq	%rbp, (%rax)		/* store %rbp there */
116	popq	%rax			/* pop off temp */
117	iretq				/* return from interrupt */
118	/*NOTREACHED*/
119
120bp_leave:
121	/*
122	 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp"
123	 * followed by a "popq %rbp".  This is quite a bit simpler on amd64
124	 * than it is on i386 -- we can exploit the fact that the %rsp is
125	 * explicitly saved to effect the pop without having to reshuffle
126	 * the other data pushed for the trap.
127	 */
128	INTR_POP
129	pushq	%rax			/* push temp */
130	movq	8(%rsp), %rax		/* load calling RIP */
131	movq	%rax, 8(%rsp)		/* store calling RIP */
132	movq	(%rbp), %rax		/* get new %rbp */
133	addq	$8, %rbp		/* adjust new %rsp */
134	movq	%rbp, 32(%rsp)		/* store new %rsp */
135	movq	%rax, %rbp		/* set new %rbp */
136	popq	%rax			/* pop off temp */
137	iretq				/* return from interrupt */
138	/*NOTREACHED*/
139
140bp_nop:
141	/* We must emulate a "nop". */
142	INTR_POP
143	iretq
144	/*NOTREACHED*/
145
146bp_ret:
147	INTR_POP
148	pushq	%rax			/* push temp */
149	movq	32(%rsp), %rax		/* load %rsp */
150	movq	(%rax), %rax		/* load calling RIP */
151	movq	%rax, 8(%rsp)		/* store calling RIP */
152	addq	$8, 32(%rsp)		/* adjust new %rsp */
153	popq	%rax			/* pop off temp */
154	iretq				/* return from interrupt */
155	/*NOTREACHED*/
156
157	END(dtrace_invop_start)
158
159/*
160void dtrace_invop_init(void)
161*/
162	ENTRY(dtrace_invop_init)
163	movq	$dtrace_invop_start, dtrace_invop_jump_addr(%rip)
164	ret
165	END(dtrace_invop_init)
166
167/*
168void dtrace_invop_uninit(void)
169*/
170	ENTRY(dtrace_invop_uninit)
171	movq	$0, dtrace_invop_jump_addr(%rip)
172	ret
173	END(dtrace_invop_uninit)
174
175/*
176greg_t dtrace_getfp(void)
177*/
178	ENTRY(dtrace_getfp)
179	movq	%rbp, %rax
180	ret
181	END(dtrace_getfp)
182
183/*
184uint32_t
185dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new)
186*/
187	ENTRY(dtrace_cas32)
188	movl	%esi, %eax
189	lock
190	cmpxchgl %edx, (%rdi)
191	ret
192	END(dtrace_cas32)
193
194/*
195void *
196dtrace_casptr(void *target, void *cmp, void *new)
197*/
198	ENTRY(dtrace_casptr)
199	movq	%rsi, %rax
200	lock
201	cmpxchgq %rdx, (%rdi)
202	ret
203	END(dtrace_casptr)
204
205/*
206uintptr_t
207dtrace_caller(int aframes)
208*/
209	ENTRY(dtrace_caller)
210	movq	$-1, %rax
211	ret
212	END(dtrace_caller)
213
214/*
215void
216dtrace_copy(uintptr_t src, uintptr_t dest, size_t size)
217*/
218	ENTRY(dtrace_copy)
219	pushq	%rbp
220	movq	%rsp, %rbp
221
222	xchgq	%rdi, %rsi		/* make %rsi source, %rdi dest */
223	movq	%rdx, %rcx		/* load count */
224	repz				/* repeat for count ... */
225	smovb				/*   move from %ds:rsi to %ed:rdi */
226	leave
227	ret
228	END(dtrace_copy)
229
230/*
231void
232dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size,
233    volatile uint16_t *flags)
234*/
235	ENTRY(dtrace_copystr)
236	pushq	%rbp
237	movq	%rsp, %rbp
238
2390:
240	movb	(%rdi), %al		/* load from source */
241	movb	%al, (%rsi)		/* store to destination */
242	addq	$1, %rdi		/* increment source pointer */
243	addq	$1, %rsi		/* increment destination pointer */
244	subq	$1, %rdx		/* decrement remaining count */
245	cmpb	$0, %al
246	je	2f
247	testq	$0xfff, %rdx		/* test if count is 4k-aligned */
248	jnz	1f			/* if not, continue with copying */
249	testq	$CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */
250	jnz	2f
2511:
252	cmpq	$0, %rdx
253	jne	0b
2542:
255	leave
256	ret
257
258	END(dtrace_copystr)
259
260/*
261uintptr_t
262dtrace_fulword(void *addr)
263*/
264	ENTRY(dtrace_fulword)
265	movq	(%rdi), %rax
266	ret
267	END(dtrace_fulword)
268
269/*
270uint8_t
271dtrace_fuword8_nocheck(void *addr)
272*/
273	ENTRY(dtrace_fuword8_nocheck)
274	xorq	%rax, %rax
275	movb	(%rdi), %al
276	ret
277	END(dtrace_fuword8_nocheck)
278
279/*
280uint16_t
281dtrace_fuword16_nocheck(void *addr)
282*/
283	ENTRY(dtrace_fuword16_nocheck)
284	xorq	%rax, %rax
285	movw	(%rdi), %ax
286	ret
287	END(dtrace_fuword16_nocheck)
288
289/*
290uint32_t
291dtrace_fuword32_nocheck(void *addr)
292*/
293	ENTRY(dtrace_fuword32_nocheck)
294	xorq	%rax, %rax
295	movl	(%rdi), %eax
296	ret
297	END(dtrace_fuword32_nocheck)
298
299/*
300uint64_t
301dtrace_fuword64_nocheck(void *addr)
302*/
303	ENTRY(dtrace_fuword64_nocheck)
304	movq	(%rdi), %rax
305	ret
306	END(dtrace_fuword64_nocheck)
307
308/*
309void
310dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
311    int fault, int fltoffs, uintptr_t illval)
312*/
313	ENTRY(dtrace_probe_error)
314	pushq	%rbp
315	movq	%rsp, %rbp
316	subq	$0x8, %rsp
317	movq	%r9, (%rsp)
318	movq	%r8, %r9
319	movq	%rcx, %r8
320	movq	%rdx, %rcx
321	movq	%rsi, %rdx
322	movq	%rdi, %rsi
323	movl	dtrace_probeid_error(%rip), %edi
324	call	dtrace_probe
325	addq	$0x8, %rsp
326	leave
327	ret
328	END(dtrace_probe_error)
329
330/*
331void
332dtrace_membar_producer(void)
333*/
334	ENTRY(dtrace_membar_producer)
335	rep;	ret	/* use 2 byte return instruction when branch target */
336			/* AMD Software Optimization Guide - Section 6.2 */
337	END(dtrace_membar_producer)
338
339/*
340void
341dtrace_membar_consumer(void)
342*/
343	ENTRY(dtrace_membar_consumer)
344	rep;	ret	/* use 2 byte return instruction when branch target */
345			/* AMD Software Optimization Guide - Section 6.2 */
346	END(dtrace_membar_consumer)
347
348/*
349dtrace_icookie_t
350dtrace_interrupt_disable(void)
351*/
352	ENTRY(dtrace_interrupt_disable)
353	pushfq
354	popq	%rax
355	cli
356	ret
357	END(dtrace_interrupt_disable)
358
359/*
360void
361dtrace_interrupt_enable(dtrace_icookie_t cookie)
362*/
363	ENTRY(dtrace_interrupt_enable)
364	pushq	%rdi
365	popfq
366	ret
367	END(dtrace_interrupt_enable)
368
369/*
370 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
371 * into the panic code implemented in panicsys().  vpanic() is responsible
372 * for passing through the format string and arguments, and constructing a
373 * regs structure on the stack into which it saves the current register
374 * values.  If we are not dying due to a fatal trap, these registers will
375 * then be preserved in panicbuf as the current processor state.  Before
376 * invoking panicsys(), vpanic() activates the first panic trigger (see
377 * common/os/panic.c) and switches to the panic_stack if successful.  Note that
378 * DTrace takes a slightly different panic path if it must panic from probe
379 * context.  Instead of calling panic, it calls into dtrace_vpanic(), which
380 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
381 * branches back into vpanic().
382 */
383
384/*
385void
386vpanic(const char *format, va_list alist)
387*/
388	ENTRY(vpanic)				/* Initial stack layout: */
389
390	pushq	%rbp				/* | %rip | 	0x60	*/
391	movq	%rsp, %rbp			/* | %rbp |	0x58	*/
392	pushfq					/* | rfl  |	0x50	*/
393	pushq	%r11				/* | %r11 |	0x48	*/
394	pushq	%r10				/* | %r10 |	0x40	*/
395	pushq	%rbx				/* | %rbx |	0x38	*/
396	pushq	%rax				/* | %rax |	0x30	*/
397	pushq	%r9				/* | %r9  |	0x28	*/
398	pushq	%r8				/* | %r8  |	0x20	*/
399	pushq	%rcx				/* | %rcx |	0x18	*/
400	pushq	%rdx				/* | %rdx |	0x10	*/
401	pushq	%rsi				/* | %rsi |	0x8 alist */
402	pushq	%rdi				/* | %rdi |	0x0 format */
403
404	movq	%rsp, %rbx			/* %rbx = current %rsp */
405
406	leaq	panic_quiesce(%rip), %rdi	/* %rdi = &panic_quiesce */
407	call	panic_trigger			/* %eax = panic_trigger() */
408
409vpanic_common:
410	/*
411	 * The panic_trigger result is in %eax from the call above, and
412	 * dtrace_panic places it in %eax before branching here.
413	 * The rdmsr instructions that follow below will clobber %eax so
414	 * we stash the panic_trigger result in %r11d.
415	 */
416	movl	%eax, %r11d
417	cmpl	$0, %r11d
418	je	0f
419
420	/*
421	 * If panic_trigger() was successful, we are the first to initiate a
422	 * panic: we now switch to the reserved panic_stack before continuing.
423	 */
424	leaq	panic_stack(%rip), %rsp
425	addq	$PANICSTKSIZE, %rsp
4260:	subq	$REGSIZE, %rsp
427	/*
428	 * Now that we've got everything set up, store the register values as
429	 * they were when we entered vpanic() to the designated location in
430	 * the regs structure we allocated on the stack.
431	 */
432#ifdef notyet
433	movq	0x0(%rbx), %rcx
434	movq	%rcx, REGOFF_RDI(%rsp)
435	movq	0x8(%rbx), %rcx
436	movq	%rcx, REGOFF_RSI(%rsp)
437	movq	0x10(%rbx), %rcx
438	movq	%rcx, REGOFF_RDX(%rsp)
439	movq	0x18(%rbx), %rcx
440	movq	%rcx, REGOFF_RCX(%rsp)
441	movq	0x20(%rbx), %rcx
442
443	movq	%rcx, REGOFF_R8(%rsp)
444	movq	0x28(%rbx), %rcx
445	movq	%rcx, REGOFF_R9(%rsp)
446	movq	0x30(%rbx), %rcx
447	movq	%rcx, REGOFF_RAX(%rsp)
448	movq	0x38(%rbx), %rcx
449	movq	%rcx, REGOFF_RBX(%rsp)
450	movq	0x58(%rbx), %rcx
451
452	movq	%rcx, REGOFF_RBP(%rsp)
453	movq	0x40(%rbx), %rcx
454	movq	%rcx, REGOFF_R10(%rsp)
455	movq	0x48(%rbx), %rcx
456	movq	%rcx, REGOFF_R11(%rsp)
457	movq	%r12, REGOFF_R12(%rsp)
458
459	movq	%r13, REGOFF_R13(%rsp)
460	movq	%r14, REGOFF_R14(%rsp)
461	movq	%r15, REGOFF_R15(%rsp)
462
463	xorl	%ecx, %ecx
464	movw	%ds, %cx
465	movq	%rcx, REGOFF_DS(%rsp)
466	movw	%es, %cx
467	movq	%rcx, REGOFF_ES(%rsp)
468	movw	%fs, %cx
469	movq	%rcx, REGOFF_FS(%rsp)
470	movw	%gs, %cx
471	movq	%rcx, REGOFF_GS(%rsp)
472
473	movq	$0, REGOFF_TRAPNO(%rsp)
474
475	movq	$0, REGOFF_ERR(%rsp)
476	leaq	vpanic(%rip), %rcx
477	movq	%rcx, REGOFF_RIP(%rsp)
478	movw	%cs, %cx
479	movzwq	%cx, %rcx
480	movq	%rcx, REGOFF_CS(%rsp)
481	movq	0x50(%rbx), %rcx
482	movq	%rcx, REGOFF_RFL(%rsp)
483	movq	%rbx, %rcx
484	addq	$0x60, %rcx
485	movq	%rcx, REGOFF_RSP(%rsp)
486	movw	%ss, %cx
487	movzwq	%cx, %rcx
488	movq	%rcx, REGOFF_SS(%rsp)
489
490	/*
491	 * panicsys(format, alist, rp, on_panic_stack)
492	 */
493	movq	REGOFF_RDI(%rsp), %rdi		/* format */
494	movq	REGOFF_RSI(%rsp), %rsi		/* alist */
495	movq	%rsp, %rdx			/* struct regs */
496	movl	%r11d, %ecx			/* on_panic_stack */
497	call	panicsys
498	addq	$REGSIZE, %rsp
499#endif
500	popq	%rdi
501	popq	%rsi
502	popq	%rdx
503	popq	%rcx
504	popq	%r8
505	popq	%r9
506	popq	%rax
507	popq	%rbx
508	popq	%r10
509	popq	%r11
510	popfq
511	leave
512	ret
513	END(vpanic)
514
515/*
516void
517dtrace_vpanic(const char *format, va_list alist)
518*/
519	ENTRY(dtrace_vpanic)			/* Initial stack layout: */
520
521	pushq	%rbp				/* | %rip | 	0x60	*/
522	movq	%rsp, %rbp			/* | %rbp |	0x58	*/
523	pushfq					/* | rfl  |	0x50	*/
524	pushq	%r11				/* | %r11 |	0x48	*/
525	pushq	%r10				/* | %r10 |	0x40	*/
526	pushq	%rbx				/* | %rbx |	0x38	*/
527	pushq	%rax				/* | %rax |	0x30	*/
528	pushq	%r9				/* | %r9  |	0x28	*/
529	pushq	%r8				/* | %r8  |	0x20	*/
530	pushq	%rcx				/* | %rcx |	0x18	*/
531	pushq	%rdx				/* | %rdx |	0x10	*/
532	pushq	%rsi				/* | %rsi |	0x8 alist */
533	pushq	%rdi				/* | %rdi |	0x0 format */
534
535	movq	%rsp, %rbx			/* %rbx = current %rsp */
536
537	leaq	panic_quiesce(%rip), %rdi	/* %rdi = &panic_quiesce */
538	call	dtrace_panic_trigger	/* %eax = dtrace_panic_trigger() */
539	jmp	vpanic_common
540
541	END(dtrace_vpanic)
542
543/*
544int
545panic_trigger(int *tp)
546*/
547	ENTRY(panic_trigger)
548	xorl	%eax, %eax
549	movl	$0xdefacedd, %edx
550	lock
551	  xchgl	%edx, (%rdi)
552	cmpl	$0, %edx
553	je	0f
554	movl	$0, %eax
555	ret
5560:	movl	$1, %eax
557	ret
558	END(panic_trigger)
559
560/*
561int
562dtrace_panic_trigger(int *tp)
563*/
564	ENTRY(dtrace_panic_trigger)
565	xorl	%eax, %eax
566	movl	$0xdefacedd, %edx
567	lock
568	  xchgl	%edx, (%rdi)
569	cmpl	$0, %edx
570	je	0f
571	movl	$0, %eax
572	ret
5730:	movl	$1, %eax
574	ret
575	END(dtrace_panic_trigger)
576