1/* $NetBSD: dtrace_asm.S,v 1.3.8.1 2012/11/22 17:39:28 riz Exp $ */ 2 3/* 4 * CDDL HEADER START 5 * 6 * The contents of this file are subject to the terms of the 7 * Common Development and Distribution License (the "License"). 8 * You may not use this file except in compliance with the License. 9 * 10 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 11 * or http://www.opensolaris.org/os/licensing. 12 * See the License for the specific language governing permissions 13 * and limitations under the License. 14 * 15 * When distributing Covered Code, include this CDDL HEADER in each 16 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 17 * If applicable, add the following below this CDDL HEADER, with the 18 * fields enclosed by brackets "[]" replaced with your own identifying 19 * information: Portions Copyright [yyyy] [name of copyright owner] 20 * 21 * CDDL HEADER END 22 * 23 * Portions Copyright 2008 John Birrell <jb@freebsd.org> 24 * 25 * $FreeBSD: src/sys/cddl/dev/dtrace/amd64/dtrace_asm.S,v 1.1.4.1 2009/08/03 08:13:06 kensmith Exp $ 26 * 27 */ 28/* 29 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 30 * Use is subject to license terms. 31 */ 32 33#define _ASM 34 35#include <sys/cpuvar_defs.h> 36#include <sys/dtrace.h> 37#include <machine/asm.h> 38#define END(a) 39#define MEXITCOUNT 40 41#include "assym.h" 42 43#define INTR_POP \ 44 MEXITCOUNT; \ 45 movq TF_RDI(%rsp),%rdi; \ 46 movq TF_RSI(%rsp),%rsi; \ 47 movq TF_RDX(%rsp),%rdx; \ 48 movq TF_RCX(%rsp),%rcx; \ 49 movq TF_R8(%rsp),%r8; \ 50 movq TF_R9(%rsp),%r9; \ 51 movq TF_RAX(%rsp),%rax; \ 52 movq TF_RBX(%rsp),%rbx; \ 53 movq TF_RBP(%rsp),%rbp; \ 54 movq TF_R10(%rsp),%r10; \ 55 movq TF_R11(%rsp),%r11; \ 56 movq TF_R12(%rsp),%r12; \ 57 movq TF_R13(%rsp),%r13; \ 58 movq TF_R14(%rsp),%r14; \ 59 movq TF_R15(%rsp),%r15; \ 60 testb $SEL_RPL_MASK,TF_CS(%rsp); \ 61 jz 1f; \ 62 cli; \ 63 swapgs; \ 641: addq $TF_RIP,%rsp; 65 66 67 ENTRY(dtrace_invop_start) 68 69 /* 70 * #BP traps with %rip set to the next address. We need to decrement 71 * the value to indicate the address of the int3 (0xcc) instruction 72 * that we substituted. 73 */ 74 movq TF_RIP(%rsp), %rdi 75 decq %rdi 76 movq TF_RSP(%rsp), %rsi 77 movq TF_RAX(%rsp), %rdx 78 pushq (%rsi) 79 movq %rsp, %rsi 80 call dtrace_invop 81 ALTENTRY(dtrace_invop_callsite) 82 addq $8, %rsp 83 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 84 je bp_push 85 cmpl $DTRACE_INVOP_LEAVE, %eax 86 je bp_leave 87 cmpl $DTRACE_INVOP_NOP, %eax 88 je bp_nop 89 cmpl $DTRACE_INVOP_RET, %eax 90 je bp_ret 91 92 /* When all else fails handle the trap in the usual way. */ 93 jmpq *dtrace_invop_calltrap_addr 94 95bp_push: 96 /* 97 * We must emulate a "pushq %rbp". To do this, we pull the stack 98 * down 8 bytes, and then store the base pointer. 99 */ 100 INTR_POP 101 subq $16, %rsp /* make room for %rbp */ 102 pushq %rax /* push temp */ 103 movq 24(%rsp), %rax /* load calling RIP */ 104 movq %rax, 8(%rsp) /* store calling RIP */ 105 movq 32(%rsp), %rax /* load calling CS */ 106 movq %rax, 16(%rsp) /* store calling CS */ 107 movq 40(%rsp), %rax /* load calling RFLAGS */ 108 movq %rax, 24(%rsp) /* store calling RFLAGS */ 109 movq 48(%rsp), %rax /* load calling RSP */ 110 subq $8, %rax /* make room for %rbp */ 111 movq %rax, 32(%rsp) /* store calling RSP */ 112 movq 56(%rsp), %rax /* load calling SS */ 113 movq %rax, 40(%rsp) /* store calling SS */ 114 movq 32(%rsp), %rax /* reload calling RSP */ 115 movq %rbp, (%rax) /* store %rbp there */ 116 popq %rax /* pop off temp */ 117 iretq /* return from interrupt */ 118 /*NOTREACHED*/ 119 120bp_leave: 121 /* 122 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" 123 * followed by a "popq %rbp". This is quite a bit simpler on amd64 124 * than it is on i386 -- we can exploit the fact that the %rsp is 125 * explicitly saved to effect the pop without having to reshuffle 126 * the other data pushed for the trap. 127 */ 128 INTR_POP 129 pushq %rax /* push temp */ 130 movq 8(%rsp), %rax /* load calling RIP */ 131 movq %rax, 8(%rsp) /* store calling RIP */ 132 movq (%rbp), %rax /* get new %rbp */ 133 addq $8, %rbp /* adjust new %rsp */ 134 movq %rbp, 32(%rsp) /* store new %rsp */ 135 movq %rax, %rbp /* set new %rbp */ 136 popq %rax /* pop off temp */ 137 iretq /* return from interrupt */ 138 /*NOTREACHED*/ 139 140bp_nop: 141 /* We must emulate a "nop". */ 142 INTR_POP 143 iretq 144 /*NOTREACHED*/ 145 146bp_ret: 147 INTR_POP 148 pushq %rax /* push temp */ 149 movq 32(%rsp), %rax /* load %rsp */ 150 movq (%rax), %rax /* load calling RIP */ 151 movq %rax, 8(%rsp) /* store calling RIP */ 152 addq $8, 32(%rsp) /* adjust new %rsp */ 153 popq %rax /* pop off temp */ 154 iretq /* return from interrupt */ 155 /*NOTREACHED*/ 156 157 END(dtrace_invop_start) 158 159/* 160void dtrace_invop_init(void) 161*/ 162 ENTRY(dtrace_invop_init) 163 movq $dtrace_invop_start, dtrace_invop_jump_addr(%rip) 164 ret 165 END(dtrace_invop_init) 166 167/* 168void dtrace_invop_uninit(void) 169*/ 170 ENTRY(dtrace_invop_uninit) 171 movq $0, dtrace_invop_jump_addr(%rip) 172 ret 173 END(dtrace_invop_uninit) 174 175/* 176greg_t dtrace_getfp(void) 177*/ 178 ENTRY(dtrace_getfp) 179 movq %rbp, %rax 180 ret 181 END(dtrace_getfp) 182 183/* 184uint32_t 185dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) 186*/ 187 ENTRY(dtrace_cas32) 188 movl %esi, %eax 189 lock 190 cmpxchgl %edx, (%rdi) 191 ret 192 END(dtrace_cas32) 193 194/* 195void * 196dtrace_casptr(void *target, void *cmp, void *new) 197*/ 198 ENTRY(dtrace_casptr) 199 movq %rsi, %rax 200 lock 201 cmpxchgq %rdx, (%rdi) 202 ret 203 END(dtrace_casptr) 204 205/* 206uintptr_t 207dtrace_caller(int aframes) 208*/ 209 ENTRY(dtrace_caller) 210 movq $-1, %rax 211 ret 212 END(dtrace_caller) 213 214/* 215void 216dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) 217*/ 218 ENTRY(dtrace_copy) 219 pushq %rbp 220 movq %rsp, %rbp 221 222 xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ 223 movq %rdx, %rcx /* load count */ 224 repz /* repeat for count ... */ 225 smovb /* move from %ds:rsi to %ed:rdi */ 226 leave 227 ret 228 END(dtrace_copy) 229 230/* 231void 232dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, 233 volatile uint16_t *flags) 234*/ 235 ENTRY(dtrace_copystr) 236 pushq %rbp 237 movq %rsp, %rbp 238 2390: 240 movb (%rdi), %al /* load from source */ 241 movb %al, (%rsi) /* store to destination */ 242 addq $1, %rdi /* increment source pointer */ 243 addq $1, %rsi /* increment destination pointer */ 244 subq $1, %rdx /* decrement remaining count */ 245 cmpb $0, %al 246 je 2f 247 testq $0xfff, %rdx /* test if count is 4k-aligned */ 248 jnz 1f /* if not, continue with copying */ 249 testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ 250 jnz 2f 2511: 252 cmpq $0, %rdx 253 jne 0b 2542: 255 leave 256 ret 257 258 END(dtrace_copystr) 259 260/* 261uintptr_t 262dtrace_fulword(void *addr) 263*/ 264 ENTRY(dtrace_fulword) 265 movq (%rdi), %rax 266 ret 267 END(dtrace_fulword) 268 269/* 270uint8_t 271dtrace_fuword8_nocheck(void *addr) 272*/ 273 ENTRY(dtrace_fuword8_nocheck) 274 xorq %rax, %rax 275 movb (%rdi), %al 276 ret 277 END(dtrace_fuword8_nocheck) 278 279/* 280uint16_t 281dtrace_fuword16_nocheck(void *addr) 282*/ 283 ENTRY(dtrace_fuword16_nocheck) 284 xorq %rax, %rax 285 movw (%rdi), %ax 286 ret 287 END(dtrace_fuword16_nocheck) 288 289/* 290uint32_t 291dtrace_fuword32_nocheck(void *addr) 292*/ 293 ENTRY(dtrace_fuword32_nocheck) 294 xorq %rax, %rax 295 movl (%rdi), %eax 296 ret 297 END(dtrace_fuword32_nocheck) 298 299/* 300uint64_t 301dtrace_fuword64_nocheck(void *addr) 302*/ 303 ENTRY(dtrace_fuword64_nocheck) 304 movq (%rdi), %rax 305 ret 306 END(dtrace_fuword64_nocheck) 307 308/* 309void 310dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, 311 int fault, int fltoffs, uintptr_t illval) 312*/ 313 ENTRY(dtrace_probe_error) 314 pushq %rbp 315 movq %rsp, %rbp 316 subq $0x8, %rsp 317 movq %r9, (%rsp) 318 movq %r8, %r9 319 movq %rcx, %r8 320 movq %rdx, %rcx 321 movq %rsi, %rdx 322 movq %rdi, %rsi 323 movl dtrace_probeid_error(%rip), %edi 324 call dtrace_probe 325 addq $0x8, %rsp 326 leave 327 ret 328 END(dtrace_probe_error) 329 330/* 331void 332dtrace_membar_producer(void) 333*/ 334 ENTRY(dtrace_membar_producer) 335 rep; ret /* use 2 byte return instruction when branch target */ 336 /* AMD Software Optimization Guide - Section 6.2 */ 337 END(dtrace_membar_producer) 338 339/* 340void 341dtrace_membar_consumer(void) 342*/ 343 ENTRY(dtrace_membar_consumer) 344 rep; ret /* use 2 byte return instruction when branch target */ 345 /* AMD Software Optimization Guide - Section 6.2 */ 346 END(dtrace_membar_consumer) 347 348/* 349dtrace_icookie_t 350dtrace_interrupt_disable(void) 351*/ 352 ENTRY(dtrace_interrupt_disable) 353 pushfq 354 popq %rax 355 cli 356 ret 357 END(dtrace_interrupt_disable) 358 359/* 360void 361dtrace_interrupt_enable(dtrace_icookie_t cookie) 362*/ 363 ENTRY(dtrace_interrupt_enable) 364 pushq %rdi 365 popfq 366 ret 367 END(dtrace_interrupt_enable) 368 369/* 370 * The panic() and cmn_err() functions invoke vpanic() as a common entry point 371 * into the panic code implemented in panicsys(). vpanic() is responsible 372 * for passing through the format string and arguments, and constructing a 373 * regs structure on the stack into which it saves the current register 374 * values. If we are not dying due to a fatal trap, these registers will 375 * then be preserved in panicbuf as the current processor state. Before 376 * invoking panicsys(), vpanic() activates the first panic trigger (see 377 * common/os/panic.c) and switches to the panic_stack if successful. Note that 378 * DTrace takes a slightly different panic path if it must panic from probe 379 * context. Instead of calling panic, it calls into dtrace_vpanic(), which 380 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 381 * branches back into vpanic(). 382 */ 383 384/* 385void 386vpanic(const char *format, va_list alist) 387*/ 388 ENTRY(vpanic) /* Initial stack layout: */ 389 390 pushq %rbp /* | %rip | 0x60 */ 391 movq %rsp, %rbp /* | %rbp | 0x58 */ 392 pushfq /* | rfl | 0x50 */ 393 pushq %r11 /* | %r11 | 0x48 */ 394 pushq %r10 /* | %r10 | 0x40 */ 395 pushq %rbx /* | %rbx | 0x38 */ 396 pushq %rax /* | %rax | 0x30 */ 397 pushq %r9 /* | %r9 | 0x28 */ 398 pushq %r8 /* | %r8 | 0x20 */ 399 pushq %rcx /* | %rcx | 0x18 */ 400 pushq %rdx /* | %rdx | 0x10 */ 401 pushq %rsi /* | %rsi | 0x8 alist */ 402 pushq %rdi /* | %rdi | 0x0 format */ 403 404 movq %rsp, %rbx /* %rbx = current %rsp */ 405 406 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 407 call panic_trigger /* %eax = panic_trigger() */ 408 409vpanic_common: 410 /* 411 * The panic_trigger result is in %eax from the call above, and 412 * dtrace_panic places it in %eax before branching here. 413 * The rdmsr instructions that follow below will clobber %eax so 414 * we stash the panic_trigger result in %r11d. 415 */ 416 movl %eax, %r11d 417 cmpl $0, %r11d 418 je 0f 419 420 /* 421 * If panic_trigger() was successful, we are the first to initiate a 422 * panic: we now switch to the reserved panic_stack before continuing. 423 */ 424 leaq panic_stack(%rip), %rsp 425 addq $PANICSTKSIZE, %rsp 4260: subq $REGSIZE, %rsp 427 /* 428 * Now that we've got everything set up, store the register values as 429 * they were when we entered vpanic() to the designated location in 430 * the regs structure we allocated on the stack. 431 */ 432#ifdef notyet 433 movq 0x0(%rbx), %rcx 434 movq %rcx, REGOFF_RDI(%rsp) 435 movq 0x8(%rbx), %rcx 436 movq %rcx, REGOFF_RSI(%rsp) 437 movq 0x10(%rbx), %rcx 438 movq %rcx, REGOFF_RDX(%rsp) 439 movq 0x18(%rbx), %rcx 440 movq %rcx, REGOFF_RCX(%rsp) 441 movq 0x20(%rbx), %rcx 442 443 movq %rcx, REGOFF_R8(%rsp) 444 movq 0x28(%rbx), %rcx 445 movq %rcx, REGOFF_R9(%rsp) 446 movq 0x30(%rbx), %rcx 447 movq %rcx, REGOFF_RAX(%rsp) 448 movq 0x38(%rbx), %rcx 449 movq %rcx, REGOFF_RBX(%rsp) 450 movq 0x58(%rbx), %rcx 451 452 movq %rcx, REGOFF_RBP(%rsp) 453 movq 0x40(%rbx), %rcx 454 movq %rcx, REGOFF_R10(%rsp) 455 movq 0x48(%rbx), %rcx 456 movq %rcx, REGOFF_R11(%rsp) 457 movq %r12, REGOFF_R12(%rsp) 458 459 movq %r13, REGOFF_R13(%rsp) 460 movq %r14, REGOFF_R14(%rsp) 461 movq %r15, REGOFF_R15(%rsp) 462 463 xorl %ecx, %ecx 464 movw %ds, %cx 465 movq %rcx, REGOFF_DS(%rsp) 466 movw %es, %cx 467 movq %rcx, REGOFF_ES(%rsp) 468 movw %fs, %cx 469 movq %rcx, REGOFF_FS(%rsp) 470 movw %gs, %cx 471 movq %rcx, REGOFF_GS(%rsp) 472 473 movq $0, REGOFF_TRAPNO(%rsp) 474 475 movq $0, REGOFF_ERR(%rsp) 476 leaq vpanic(%rip), %rcx 477 movq %rcx, REGOFF_RIP(%rsp) 478 movw %cs, %cx 479 movzwq %cx, %rcx 480 movq %rcx, REGOFF_CS(%rsp) 481 movq 0x50(%rbx), %rcx 482 movq %rcx, REGOFF_RFL(%rsp) 483 movq %rbx, %rcx 484 addq $0x60, %rcx 485 movq %rcx, REGOFF_RSP(%rsp) 486 movw %ss, %cx 487 movzwq %cx, %rcx 488 movq %rcx, REGOFF_SS(%rsp) 489 490 /* 491 * panicsys(format, alist, rp, on_panic_stack) 492 */ 493 movq REGOFF_RDI(%rsp), %rdi /* format */ 494 movq REGOFF_RSI(%rsp), %rsi /* alist */ 495 movq %rsp, %rdx /* struct regs */ 496 movl %r11d, %ecx /* on_panic_stack */ 497 call panicsys 498 addq $REGSIZE, %rsp 499#endif 500 popq %rdi 501 popq %rsi 502 popq %rdx 503 popq %rcx 504 popq %r8 505 popq %r9 506 popq %rax 507 popq %rbx 508 popq %r10 509 popq %r11 510 popfq 511 leave 512 ret 513 END(vpanic) 514 515/* 516void 517dtrace_vpanic(const char *format, va_list alist) 518*/ 519 ENTRY(dtrace_vpanic) /* Initial stack layout: */ 520 521 pushq %rbp /* | %rip | 0x60 */ 522 movq %rsp, %rbp /* | %rbp | 0x58 */ 523 pushfq /* | rfl | 0x50 */ 524 pushq %r11 /* | %r11 | 0x48 */ 525 pushq %r10 /* | %r10 | 0x40 */ 526 pushq %rbx /* | %rbx | 0x38 */ 527 pushq %rax /* | %rax | 0x30 */ 528 pushq %r9 /* | %r9 | 0x28 */ 529 pushq %r8 /* | %r8 | 0x20 */ 530 pushq %rcx /* | %rcx | 0x18 */ 531 pushq %rdx /* | %rdx | 0x10 */ 532 pushq %rsi /* | %rsi | 0x8 alist */ 533 pushq %rdi /* | %rdi | 0x0 format */ 534 535 movq %rsp, %rbx /* %rbx = current %rsp */ 536 537 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 538 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */ 539 jmp vpanic_common 540 541 END(dtrace_vpanic) 542 543/* 544int 545panic_trigger(int *tp) 546*/ 547 ENTRY(panic_trigger) 548 xorl %eax, %eax 549 movl $0xdefacedd, %edx 550 lock 551 xchgl %edx, (%rdi) 552 cmpl $0, %edx 553 je 0f 554 movl $0, %eax 555 ret 5560: movl $1, %eax 557 ret 558 END(panic_trigger) 559 560/* 561int 562dtrace_panic_trigger(int *tp) 563*/ 564 ENTRY(dtrace_panic_trigger) 565 xorl %eax, %eax 566 movl $0xdefacedd, %edx 567 lock 568 xchgl %edx, (%rdi) 569 cmpl $0, %edx 570 je 0f 571 movl $0, %eax 572 ret 5730: movl $1, %eax 574 ret 575 END(dtrace_panic_trigger) 576