dtrace_asm.S revision 1.3
1/* $NetBSD: dtrace_asm.S,v 1.3 2010/03/13 22:31:15 christos Exp $ */ 2 3/* 4 * CDDL HEADER START 5 * 6 * The contents of this file are subject to the terms of the 7 * Common Development and Distribution License (the "License"). 8 * You may not use this file except in compliance with the License. 9 * 10 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 11 * or http://www.opensolaris.org/os/licensing. 12 * See the License for the specific language governing permissions 13 * and limitations under the License. 14 * 15 * When distributing Covered Code, include this CDDL HEADER in each 16 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 17 * If applicable, add the following below this CDDL HEADER, with the 18 * fields enclosed by brackets "[]" replaced with your own identifying 19 * information: Portions Copyright [yyyy] [name of copyright owner] 20 * 21 * CDDL HEADER END 22 * 23 * Portions Copyright 2008 John Birrell <jb@freebsd.org> 24 * 25 * $FreeBSD: src/sys/cddl/dev/dtrace/amd64/dtrace_asm.S,v 1.1.4.1 2009/08/03 08:13:06 kensmith Exp $ 26 * 27 */ 28/* 29 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 30 * Use is subject to license terms. 31 */ 32 33#define _ASM 34 35 36#include <sys/cpuvar_defs.h> 37#include <sys/dtrace.h> 38#include <machine/asm.h> 39#define END(a) 40#define MEXITCOUNT 41 42#include "assym.h" 43 44#define INTR_POP \ 45 MEXITCOUNT; \ 46 movq TF_RDI(%rsp),%rdi; \ 47 movq TF_RSI(%rsp),%rsi; \ 48 movq TF_RDX(%rsp),%rdx; \ 49 movq TF_RCX(%rsp),%rcx; \ 50 movq TF_R8(%rsp),%r8; \ 51 movq TF_R9(%rsp),%r9; \ 52 movq TF_RAX(%rsp),%rax; \ 53 movq TF_RBX(%rsp),%rbx; \ 54 movq TF_RBP(%rsp),%rbp; \ 55 movq TF_R10(%rsp),%r10; \ 56 movq TF_R11(%rsp),%r11; \ 57 movq TF_R12(%rsp),%r12; \ 58 movq TF_R13(%rsp),%r13; \ 59 movq TF_R14(%rsp),%r14; \ 60 movq TF_R15(%rsp),%r15; \ 61 testb $SEL_RPL_MASK,TF_CS(%rsp); \ 62 jz 1f; \ 63 cli; \ 64 swapgs; \ 651: addq $TF_RIP,%rsp; 66 67 68 .globl calltrap 69 .type calltrap,@function 70 ENTRY(dtrace_invop_start) 71 72 /* 73 * #BP traps with %rip set to the next address. We need to decrement 74 * the value to indicate the address of the int3 (0xcc) instruction 75 * that we substituted. 76 */ 77 movq TF_RIP(%rsp), %rdi 78 decq %rdi 79 movq TF_RSP(%rsp), %rsi 80 movq TF_RAX(%rsp), %rdx 81 pushq (%rsi) 82 movq %rsp, %rsi 83 call dtrace_invop 84// ALTENTRY(dtrace_invop_callsite) 85 addq $8, %rsp 86 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 87 je bp_push 88 cmpl $DTRACE_INVOP_LEAVE, %eax 89 je bp_leave 90 cmpl $DTRACE_INVOP_NOP, %eax 91 je bp_nop 92 cmpl $DTRACE_INVOP_RET, %eax 93 je bp_ret 94 95 /* When all else fails handle the trap in the usual way. */ 96 jmpq *dtrace_invop_calltrap_addr 97 98bp_push: 99 /* 100 * We must emulate a "pushq %rbp". To do this, we pull the stack 101 * down 8 bytes, and then store the base pointer. 102 */ 103 INTR_POP 104 subq $16, %rsp /* make room for %rbp */ 105 pushq %rax /* push temp */ 106 movq 24(%rsp), %rax /* load calling RIP */ 107 movq %rax, 8(%rsp) /* store calling RIP */ 108 movq 32(%rsp), %rax /* load calling CS */ 109 movq %rax, 16(%rsp) /* store calling CS */ 110 movq 40(%rsp), %rax /* load calling RFLAGS */ 111 movq %rax, 24(%rsp) /* store calling RFLAGS */ 112 movq 48(%rsp), %rax /* load calling RSP */ 113 subq $8, %rax /* make room for %rbp */ 114 movq %rax, 32(%rsp) /* store calling RSP */ 115 movq 56(%rsp), %rax /* load calling SS */ 116 movq %rax, 40(%rsp) /* store calling SS */ 117 movq 32(%rsp), %rax /* reload calling RSP */ 118 movq %rbp, (%rax) /* store %rbp there */ 119 popq %rax /* pop off temp */ 120 iretq /* return from interrupt */ 121 /*NOTREACHED*/ 122 123bp_leave: 124 /* 125 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" 126 * followed by a "popq %rbp". This is quite a bit simpler on amd64 127 * than it is on i386 -- we can exploit the fact that the %rsp is 128 * explicitly saved to effect the pop without having to reshuffle 129 * the other data pushed for the trap. 130 */ 131 INTR_POP 132 pushq %rax /* push temp */ 133 movq 8(%rsp), %rax /* load calling RIP */ 134 movq %rax, 8(%rsp) /* store calling RIP */ 135 movq (%rbp), %rax /* get new %rbp */ 136 addq $8, %rbp /* adjust new %rsp */ 137 movq %rbp, 32(%rsp) /* store new %rsp */ 138 movq %rax, %rbp /* set new %rbp */ 139 popq %rax /* pop off temp */ 140 iretq /* return from interrupt */ 141 /*NOTREACHED*/ 142 143bp_nop: 144 /* We must emulate a "nop". */ 145 INTR_POP 146 iretq 147 /*NOTREACHED*/ 148 149bp_ret: 150// INTR_POP 151 pushq %rax /* push temp */ 152 movq 32(%rsp), %rax /* load %rsp */ 153 movq (%rax), %rax /* load calling RIP */ 154 movq %rax, 8(%rsp) /* store calling RIP */ 155 addq $8, 32(%rsp) /* adjust new %rsp */ 156 popq %rax /* pop off temp */ 157 iretq /* return from interrupt */ 158 /*NOTREACHED*/ 159 160 END(dtrace_invop_start) 161 162/* 163void dtrace_invop_init(void) 164*/ 165 ENTRY(dtrace_invop_init) 166 movq $dtrace_invop_start, dtrace_invop_jump_addr(%rip) 167 ret 168 END(dtrace_invop_init) 169 170/* 171void dtrace_invop_uninit(void) 172*/ 173 ENTRY(dtrace_invop_uninit) 174 movq $0, dtrace_invop_jump_addr(%rip) 175 ret 176 END(dtrace_invop_uninit) 177 178/* 179greg_t dtrace_getfp(void) 180*/ 181 ENTRY(dtrace_getfp) 182 movq %rbp, %rax 183 ret 184 END(dtrace_getfp) 185 186/* 187uint32_t 188dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) 189*/ 190 ENTRY(dtrace_cas32) 191 movl %esi, %eax 192 lock 193 cmpxchgl %edx, (%rdi) 194 ret 195 END(dtrace_cas32) 196 197/* 198void * 199dtrace_casptr(void *target, void *cmp, void *new) 200*/ 201 ENTRY(dtrace_casptr) 202 movq %rsi, %rax 203 lock 204 cmpxchgq %rdx, (%rdi) 205 ret 206 END(dtrace_casptr) 207 208/* 209uintptr_t 210dtrace_caller(int aframes) 211*/ 212 ENTRY(dtrace_caller) 213 movq $-1, %rax 214 ret 215 END(dtrace_caller) 216 217/* 218void 219dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) 220*/ 221 ENTRY(dtrace_copy) 222 pushq %rbp 223 movq %rsp, %rbp 224 225 xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ 226 movq %rdx, %rcx /* load count */ 227 repz /* repeat for count ... */ 228 smovb /* move from %ds:rsi to %ed:rdi */ 229 leave 230 ret 231 END(dtrace_copy) 232 233/* 234void 235dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, 236 volatile uint16_t *flags) 237*/ 238 ENTRY(dtrace_copystr) 239 pushq %rbp 240 movq %rsp, %rbp 241 2420: 243 movb (%rdi), %al /* load from source */ 244 movb %al, (%rsi) /* store to destination */ 245 addq $1, %rdi /* increment source pointer */ 246 addq $1, %rsi /* increment destination pointer */ 247 subq $1, %rdx /* decrement remaining count */ 248 cmpb $0, %al 249 je 2f 250 testq $0xfff, %rdx /* test if count is 4k-aligned */ 251 jnz 1f /* if not, continue with copying */ 252 testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ 253 jnz 2f 2541: 255 cmpq $0, %rdx 256 jne 0b 2572: 258 leave 259 ret 260 261 END(dtrace_copystr) 262 263/* 264uintptr_t 265dtrace_fulword(void *addr) 266*/ 267 ENTRY(dtrace_fulword) 268 movq (%rdi), %rax 269 ret 270 END(dtrace_fulword) 271 272/* 273uint8_t 274dtrace_fuword8_nocheck(void *addr) 275*/ 276 ENTRY(dtrace_fuword8_nocheck) 277 xorq %rax, %rax 278 movb (%rdi), %al 279 ret 280 END(dtrace_fuword8_nocheck) 281 282/* 283uint16_t 284dtrace_fuword16_nocheck(void *addr) 285*/ 286 ENTRY(dtrace_fuword16_nocheck) 287 xorq %rax, %rax 288 movw (%rdi), %ax 289 ret 290 END(dtrace_fuword16_nocheck) 291 292/* 293uint32_t 294dtrace_fuword32_nocheck(void *addr) 295*/ 296 ENTRY(dtrace_fuword32_nocheck) 297 xorq %rax, %rax 298 movl (%rdi), %eax 299 ret 300 END(dtrace_fuword32_nocheck) 301 302/* 303uint64_t 304dtrace_fuword64_nocheck(void *addr) 305*/ 306 ENTRY(dtrace_fuword64_nocheck) 307 movq (%rdi), %rax 308 ret 309 END(dtrace_fuword64_nocheck) 310 311/* 312void 313dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, 314 int fault, int fltoffs, uintptr_t illval) 315*/ 316 ENTRY(dtrace_probe_error) 317 pushq %rbp 318 movq %rsp, %rbp 319 subq $0x8, %rsp 320 movq %r9, (%rsp) 321 movq %r8, %r9 322 movq %rcx, %r8 323 movq %rdx, %rcx 324 movq %rsi, %rdx 325 movq %rdi, %rsi 326 movl dtrace_probeid_error(%rip), %edi 327 call dtrace_probe 328 addq $0x8, %rsp 329 leave 330 ret 331 END(dtrace_probe_error) 332 333/* 334void 335dtrace_membar_producer(void) 336*/ 337 ENTRY(dtrace_membar_producer) 338 rep; ret /* use 2 byte return instruction when branch target */ 339 /* AMD Software Optimization Guide - Section 6.2 */ 340 END(dtrace_membar_producer) 341 342/* 343void 344dtrace_membar_consumer(void) 345*/ 346 ENTRY(dtrace_membar_consumer) 347 rep; ret /* use 2 byte return instruction when branch target */ 348 /* AMD Software Optimization Guide - Section 6.2 */ 349 END(dtrace_membar_consumer) 350 351/* 352dtrace_icookie_t 353dtrace_interrupt_disable(void) 354*/ 355 ENTRY(dtrace_interrupt_disable) 356 pushfq 357 popq %rax 358 cli 359 ret 360 END(dtrace_interrupt_disable) 361 362/* 363void 364dtrace_interrupt_enable(dtrace_icookie_t cookie) 365*/ 366 ENTRY(dtrace_interrupt_enable) 367 pushq %rdi 368 popfq 369 ret 370 END(dtrace_interrupt_enable) 371 372/* 373 * The panic() and cmn_err() functions invoke vpanic() as a common entry point 374 * into the panic code implemented in panicsys(). vpanic() is responsible 375 * for passing through the format string and arguments, and constructing a 376 * regs structure on the stack into which it saves the current register 377 * values. If we are not dying due to a fatal trap, these registers will 378 * then be preserved in panicbuf as the current processor state. Before 379 * invoking panicsys(), vpanic() activates the first panic trigger (see 380 * common/os/panic.c) and switches to the panic_stack if successful. Note that 381 * DTrace takes a slightly different panic path if it must panic from probe 382 * context. Instead of calling panic, it calls into dtrace_vpanic(), which 383 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 384 * branches back into vpanic(). 385 */ 386 387/* 388void 389vpanic(const char *format, va_list alist) 390*/ 391 ENTRY(vpanic) /* Initial stack layout: */ 392 393 pushq %rbp /* | %rip | 0x60 */ 394 movq %rsp, %rbp /* | %rbp | 0x58 */ 395 pushfq /* | rfl | 0x50 */ 396 pushq %r11 /* | %r11 | 0x48 */ 397 pushq %r10 /* | %r10 | 0x40 */ 398 pushq %rbx /* | %rbx | 0x38 */ 399 pushq %rax /* | %rax | 0x30 */ 400 pushq %r9 /* | %r9 | 0x28 */ 401 pushq %r8 /* | %r8 | 0x20 */ 402 pushq %rcx /* | %rcx | 0x18 */ 403 pushq %rdx /* | %rdx | 0x10 */ 404 pushq %rsi /* | %rsi | 0x8 alist */ 405 pushq %rdi /* | %rdi | 0x0 format */ 406 407 movq %rsp, %rbx /* %rbx = current %rsp */ 408 409 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 410 call panic_trigger /* %eax = panic_trigger() */ 411 412vpanic_common: 413 /* 414 * The panic_trigger result is in %eax from the call above, and 415 * dtrace_panic places it in %eax before branching here. 416 * The rdmsr instructions that follow below will clobber %eax so 417 * we stash the panic_trigger result in %r11d. 418 */ 419 movl %eax, %r11d 420 cmpl $0, %r11d 421 je 0f 422 423 /* 424 * If panic_trigger() was successful, we are the first to initiate a 425 * panic: we now switch to the reserved panic_stack before continuing. 426 */ 427 leaq panic_stack(%rip), %rsp 428 addq $PANICSTKSIZE, %rsp 4290: subq $REGSIZE, %rsp 430 /* 431 * Now that we've got everything set up, store the register values as 432 * they were when we entered vpanic() to the designated location in 433 * the regs structure we allocated on the stack. 434 */ 435#ifdef notyet 436 movq 0x0(%rbx), %rcx 437 movq %rcx, REGOFF_RDI(%rsp) 438 movq 0x8(%rbx), %rcx 439 movq %rcx, REGOFF_RSI(%rsp) 440 movq 0x10(%rbx), %rcx 441 movq %rcx, REGOFF_RDX(%rsp) 442 movq 0x18(%rbx), %rcx 443 movq %rcx, REGOFF_RCX(%rsp) 444 movq 0x20(%rbx), %rcx 445 446 movq %rcx, REGOFF_R8(%rsp) 447 movq 0x28(%rbx), %rcx 448 movq %rcx, REGOFF_R9(%rsp) 449 movq 0x30(%rbx), %rcx 450 movq %rcx, REGOFF_RAX(%rsp) 451 movq 0x38(%rbx), %rcx 452 movq %rcx, REGOFF_RBX(%rsp) 453 movq 0x58(%rbx), %rcx 454 455 movq %rcx, REGOFF_RBP(%rsp) 456 movq 0x40(%rbx), %rcx 457 movq %rcx, REGOFF_R10(%rsp) 458 movq 0x48(%rbx), %rcx 459 movq %rcx, REGOFF_R11(%rsp) 460 movq %r12, REGOFF_R12(%rsp) 461 462 movq %r13, REGOFF_R13(%rsp) 463 movq %r14, REGOFF_R14(%rsp) 464 movq %r15, REGOFF_R15(%rsp) 465 466 xorl %ecx, %ecx 467 movw %ds, %cx 468 movq %rcx, REGOFF_DS(%rsp) 469 movw %es, %cx 470 movq %rcx, REGOFF_ES(%rsp) 471 movw %fs, %cx 472 movq %rcx, REGOFF_FS(%rsp) 473 movw %gs, %cx 474 movq %rcx, REGOFF_GS(%rsp) 475 476 movq $0, REGOFF_TRAPNO(%rsp) 477 478 movq $0, REGOFF_ERR(%rsp) 479 leaq vpanic(%rip), %rcx 480 movq %rcx, REGOFF_RIP(%rsp) 481 movw %cs, %cx 482 movzwq %cx, %rcx 483 movq %rcx, REGOFF_CS(%rsp) 484 movq 0x50(%rbx), %rcx 485 movq %rcx, REGOFF_RFL(%rsp) 486 movq %rbx, %rcx 487 addq $0x60, %rcx 488 movq %rcx, REGOFF_RSP(%rsp) 489 movw %ss, %cx 490 movzwq %cx, %rcx 491 movq %rcx, REGOFF_SS(%rsp) 492 493 /* 494 * panicsys(format, alist, rp, on_panic_stack) 495 */ 496 movq REGOFF_RDI(%rsp), %rdi /* format */ 497 movq REGOFF_RSI(%rsp), %rsi /* alist */ 498 movq %rsp, %rdx /* struct regs */ 499 movl %r11d, %ecx /* on_panic_stack */ 500 call panicsys 501 addq $REGSIZE, %rsp 502#endif 503 popq %rdi 504 popq %rsi 505 popq %rdx 506 popq %rcx 507 popq %r8 508 popq %r9 509 popq %rax 510 popq %rbx 511 popq %r10 512 popq %r11 513 popfq 514 leave 515 ret 516 END(vpanic) 517 518/* 519void 520dtrace_vpanic(const char *format, va_list alist) 521*/ 522 ENTRY(dtrace_vpanic) /* Initial stack layout: */ 523 524 pushq %rbp /* | %rip | 0x60 */ 525 movq %rsp, %rbp /* | %rbp | 0x58 */ 526 pushfq /* | rfl | 0x50 */ 527 pushq %r11 /* | %r11 | 0x48 */ 528 pushq %r10 /* | %r10 | 0x40 */ 529 pushq %rbx /* | %rbx | 0x38 */ 530 pushq %rax /* | %rax | 0x30 */ 531 pushq %r9 /* | %r9 | 0x28 */ 532 pushq %r8 /* | %r8 | 0x20 */ 533 pushq %rcx /* | %rcx | 0x18 */ 534 pushq %rdx /* | %rdx | 0x10 */ 535 pushq %rsi /* | %rsi | 0x8 alist */ 536 pushq %rdi /* | %rdi | 0x0 format */ 537 538 movq %rsp, %rbx /* %rbx = current %rsp */ 539 540 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 541 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */ 542 jmp vpanic_common 543 544 END(dtrace_vpanic) 545 546/* 547int 548panic_trigger(int *tp) 549*/ 550 ENTRY(panic_trigger) 551 xorl %eax, %eax 552 movl $0xdefacedd, %edx 553 lock 554 xchgl %edx, (%rdi) 555 cmpl $0, %edx 556 je 0f 557 movl $0, %eax 558 ret 5590: movl $1, %eax 560 ret 561 END(panic_trigger) 562 563/* 564int 565dtrace_panic_trigger(int *tp) 566*/ 567 ENTRY(dtrace_panic_trigger) 568 xorl %eax, %eax 569 movl $0xdefacedd, %edx 570 lock 571 xchgl %edx, (%rdi) 572 cmpl $0, %edx 573 je 0f 574 movl $0, %eax 575 ret 5760: movl $1, %eax 577 ret 578 END(dtrace_panic_trigger) 579