1193323Sed//===-- X86JITInfo.cpp - Implement the JIT interfaces for the X86 target --===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This file implements the JIT interfaces for the X86 target. 11193323Sed// 12193323Sed//===----------------------------------------------------------------------===// 13193323Sed 14193323Sed#define DEBUG_TYPE "jit" 15193323Sed#include "X86JITInfo.h" 16193323Sed#include "X86Relocations.h" 17193323Sed#include "X86Subtarget.h" 18198090Srdivacky#include "X86TargetMachine.h" 19252723Sdim#include "llvm/IR/Function.h" 20193323Sed#include "llvm/Support/Compiler.h" 21198090Srdivacky#include "llvm/Support/ErrorHandling.h" 22218893Sdim#include "llvm/Support/Valgrind.h" 23193323Sed#include <cstdlib> 24193323Sed#include <cstring> 25193323Sedusing namespace llvm; 26193323Sed 27193323Sed// Determine the platform we're running on 28198090Srdivacky#if defined (__x86_64__) || defined (_M_AMD64) || defined (_M_X64) 29193323Sed# define X86_64_JIT 30193323Sed#elif defined(__i386__) || defined(i386) || defined(_M_IX86) 31193323Sed# define X86_32_JIT 32193323Sed#endif 33193323Sed 34193323Sedvoid X86JITInfo::replaceMachineCodeForFunction(void *Old, void *New) { 35193323Sed unsigned char *OldByte = (unsigned char *)Old; 36193323Sed *OldByte++ = 0xE9; // Emit JMP opcode. 37193323Sed unsigned *OldWord = (unsigned *)OldByte; 38193323Sed unsigned NewAddr = (intptr_t)New; 39193323Sed unsigned OldAddr = (intptr_t)OldWord; 40193323Sed *OldWord = NewAddr - OldAddr - 4; // Emit PC-relative addr of New code. 41205218Srdivacky 42205218Srdivacky // X86 doesn't need to invalidate the processor cache, so just invalidate 43205218Srdivacky // Valgrind's cache directly. 44205218Srdivacky sys::ValgrindDiscardTranslations(Old, 5); 45193323Sed} 46193323Sed 47193323Sed 48193323Sed/// JITCompilerFunction - This contains the address of the JIT function used to 49193323Sed/// compile a function lazily. 50193323Sedstatic TargetJITInfo::JITCompilerFn JITCompilerFunction; 51193323Sed 52193323Sed// Get the ASMPREFIX for the current host. This is often '_'. 53193323Sed#ifndef __USER_LABEL_PREFIX__ 54193323Sed#define __USER_LABEL_PREFIX__ 55193323Sed#endif 56193323Sed#define GETASMPREFIX2(X) #X 57193323Sed#define GETASMPREFIX(X) GETASMPREFIX2(X) 58193323Sed#define ASMPREFIX GETASMPREFIX(__USER_LABEL_PREFIX__) 59193323Sed 60193323Sed// For ELF targets, use a .size and .type directive, to let tools 61193323Sed// know the extent of functions defined in assembler. 62193323Sed#if defined(__ELF__) 63193323Sed# define SIZE(sym) ".size " #sym ", . - " #sym "\n" 64193323Sed# define TYPE_FUNCTION(sym) ".type " #sym ", @function\n" 65193323Sed#else 66193323Sed# define SIZE(sym) 67193323Sed# define TYPE_FUNCTION(sym) 68193323Sed#endif 69193323Sed 70193323Sed// Provide a convenient way for disabling usage of CFI directives. 71193323Sed// This is needed for old/broken assemblers (for example, gas on 72193323Sed// Darwin is pretty old and doesn't support these directives) 73193323Sed#if defined(__APPLE__) 74193323Sed# define CFI(x) 75193323Sed#else 76193323Sed// FIXME: Disable this until we really want to use it. Also, we will 77193323Sed// need to add some workarounds for compilers, which support 78193323Sed// only subset of these directives. 79193323Sed# define CFI(x) 80193323Sed#endif 81193323Sed 82252723Sdim// Provide a wrapper for LLVMX86CompilationCallback2 that saves non-traditional 83193323Sed// callee saved registers, for the fastcc calling convention. 84193323Sedextern "C" { 85193323Sed#if defined(X86_64_JIT) 86193323Sed# ifndef _MSC_VER 87193323Sed // No need to save EAX/EDX for X86-64. 88193323Sed void X86CompilationCallback(void); 89193323Sed asm( 90193323Sed ".text\n" 91193323Sed ".align 8\n" 92193323Sed ".globl " ASMPREFIX "X86CompilationCallback\n" 93193323Sed TYPE_FUNCTION(X86CompilationCallback) 94193323Sed ASMPREFIX "X86CompilationCallback:\n" 95193323Sed CFI(".cfi_startproc\n") 96193323Sed // Save RBP 97193323Sed "pushq %rbp\n" 98193323Sed CFI(".cfi_def_cfa_offset 16\n") 99193323Sed CFI(".cfi_offset %rbp, -16\n") 100193323Sed // Save RSP 101193323Sed "movq %rsp, %rbp\n" 102193323Sed CFI(".cfi_def_cfa_register %rbp\n") 103193323Sed // Save all int arg registers 104193323Sed "pushq %rdi\n" 105193323Sed CFI(".cfi_rel_offset %rdi, 0\n") 106193323Sed "pushq %rsi\n" 107193323Sed CFI(".cfi_rel_offset %rsi, 8\n") 108193323Sed "pushq %rdx\n" 109193323Sed CFI(".cfi_rel_offset %rdx, 16\n") 110193323Sed "pushq %rcx\n" 111193323Sed CFI(".cfi_rel_offset %rcx, 24\n") 112193323Sed "pushq %r8\n" 113193323Sed CFI(".cfi_rel_offset %r8, 32\n") 114193323Sed "pushq %r9\n" 115193323Sed CFI(".cfi_rel_offset %r9, 40\n") 116193323Sed // Align stack on 16-byte boundary. ESP might not be properly aligned 117193323Sed // (8 byte) if this is called from an indirect stub. 118193323Sed "andq $-16, %rsp\n" 119193323Sed // Save all XMM arg registers 120193323Sed "subq $128, %rsp\n" 121193323Sed "movaps %xmm0, (%rsp)\n" 122193323Sed "movaps %xmm1, 16(%rsp)\n" 123193323Sed "movaps %xmm2, 32(%rsp)\n" 124193323Sed "movaps %xmm3, 48(%rsp)\n" 125193323Sed "movaps %xmm4, 64(%rsp)\n" 126193323Sed "movaps %xmm5, 80(%rsp)\n" 127193323Sed "movaps %xmm6, 96(%rsp)\n" 128193323Sed "movaps %xmm7, 112(%rsp)\n" 129193323Sed // JIT callee 130263509Sdim#if defined(_WIN64) || defined(__CYGWIN__) 131218893Sdim "subq $32, %rsp\n" 132218893Sdim "movq %rbp, %rcx\n" // Pass prev frame and return address 133218893Sdim "movq 8(%rbp), %rdx\n" 134252723Sdim "call " ASMPREFIX "LLVMX86CompilationCallback2\n" 135218893Sdim "addq $32, %rsp\n" 136218893Sdim#else 137193323Sed "movq %rbp, %rdi\n" // Pass prev frame and return address 138193323Sed "movq 8(%rbp), %rsi\n" 139252723Sdim "call " ASMPREFIX "LLVMX86CompilationCallback2\n" 140218893Sdim#endif 141193323Sed // Restore all XMM arg registers 142193323Sed "movaps 112(%rsp), %xmm7\n" 143193323Sed "movaps 96(%rsp), %xmm6\n" 144193323Sed "movaps 80(%rsp), %xmm5\n" 145193323Sed "movaps 64(%rsp), %xmm4\n" 146193323Sed "movaps 48(%rsp), %xmm3\n" 147193323Sed "movaps 32(%rsp), %xmm2\n" 148193323Sed "movaps 16(%rsp), %xmm1\n" 149193323Sed "movaps (%rsp), %xmm0\n" 150193323Sed // Restore RSP 151193323Sed "movq %rbp, %rsp\n" 152193323Sed CFI(".cfi_def_cfa_register %rsp\n") 153193323Sed // Restore all int arg registers 154193323Sed "subq $48, %rsp\n" 155193323Sed CFI(".cfi_adjust_cfa_offset 48\n") 156193323Sed "popq %r9\n" 157193323Sed CFI(".cfi_adjust_cfa_offset -8\n") 158193323Sed CFI(".cfi_restore %r9\n") 159193323Sed "popq %r8\n" 160193323Sed CFI(".cfi_adjust_cfa_offset -8\n") 161193323Sed CFI(".cfi_restore %r8\n") 162193323Sed "popq %rcx\n" 163193323Sed CFI(".cfi_adjust_cfa_offset -8\n") 164193323Sed CFI(".cfi_restore %rcx\n") 165193323Sed "popq %rdx\n" 166193323Sed CFI(".cfi_adjust_cfa_offset -8\n") 167193323Sed CFI(".cfi_restore %rdx\n") 168193323Sed "popq %rsi\n" 169193323Sed CFI(".cfi_adjust_cfa_offset -8\n") 170193323Sed CFI(".cfi_restore %rsi\n") 171193323Sed "popq %rdi\n" 172193323Sed CFI(".cfi_adjust_cfa_offset -8\n") 173193323Sed CFI(".cfi_restore %rdi\n") 174193323Sed // Restore RBP 175193323Sed "popq %rbp\n" 176193323Sed CFI(".cfi_adjust_cfa_offset -8\n") 177193323Sed CFI(".cfi_restore %rbp\n") 178193323Sed "ret\n" 179193323Sed CFI(".cfi_endproc\n") 180193323Sed SIZE(X86CompilationCallback) 181193323Sed ); 182193323Sed# else 183193323Sed // No inline assembler support on this platform. The routine is in external 184193323Sed // file. 185193323Sed void X86CompilationCallback(); 186193323Sed 187193323Sed# endif 188193323Sed#elif defined (X86_32_JIT) 189193323Sed# ifndef _MSC_VER 190193323Sed void X86CompilationCallback(void); 191193323Sed asm( 192193323Sed ".text\n" 193193323Sed ".align 8\n" 194193323Sed ".globl " ASMPREFIX "X86CompilationCallback\n" 195193323Sed TYPE_FUNCTION(X86CompilationCallback) 196193323Sed ASMPREFIX "X86CompilationCallback:\n" 197193323Sed CFI(".cfi_startproc\n") 198193323Sed "pushl %ebp\n" 199193323Sed CFI(".cfi_def_cfa_offset 8\n") 200193323Sed CFI(".cfi_offset %ebp, -8\n") 201193323Sed "movl %esp, %ebp\n" // Standard prologue 202193323Sed CFI(".cfi_def_cfa_register %ebp\n") 203193323Sed "pushl %eax\n" 204193323Sed CFI(".cfi_rel_offset %eax, 0\n") 205193323Sed "pushl %edx\n" // Save EAX/EDX/ECX 206193323Sed CFI(".cfi_rel_offset %edx, 4\n") 207193323Sed "pushl %ecx\n" 208193323Sed CFI(".cfi_rel_offset %ecx, 8\n") 209193323Sed# if defined(__APPLE__) 210193323Sed "andl $-16, %esp\n" // Align ESP on 16-byte boundary 211193323Sed# endif 212193323Sed "subl $16, %esp\n" 213193323Sed "movl 4(%ebp), %eax\n" // Pass prev frame and return address 214193323Sed "movl %eax, 4(%esp)\n" 215193323Sed "movl %ebp, (%esp)\n" 216252723Sdim "call " ASMPREFIX "LLVMX86CompilationCallback2\n" 217193323Sed "movl %ebp, %esp\n" // Restore ESP 218193323Sed CFI(".cfi_def_cfa_register %esp\n") 219193323Sed "subl $12, %esp\n" 220193323Sed CFI(".cfi_adjust_cfa_offset 12\n") 221193323Sed "popl %ecx\n" 222193323Sed CFI(".cfi_adjust_cfa_offset -4\n") 223193323Sed CFI(".cfi_restore %ecx\n") 224193323Sed "popl %edx\n" 225193323Sed CFI(".cfi_adjust_cfa_offset -4\n") 226193323Sed CFI(".cfi_restore %edx\n") 227193323Sed "popl %eax\n" 228193323Sed CFI(".cfi_adjust_cfa_offset -4\n") 229193323Sed CFI(".cfi_restore %eax\n") 230193323Sed "popl %ebp\n" 231193323Sed CFI(".cfi_adjust_cfa_offset -4\n") 232193323Sed CFI(".cfi_restore %ebp\n") 233193323Sed "ret\n" 234193323Sed CFI(".cfi_endproc\n") 235193323Sed SIZE(X86CompilationCallback) 236193323Sed ); 237193323Sed 238193323Sed // Same as X86CompilationCallback but also saves XMM argument registers. 239193323Sed void X86CompilationCallback_SSE(void); 240193323Sed asm( 241193323Sed ".text\n" 242193323Sed ".align 8\n" 243193323Sed ".globl " ASMPREFIX "X86CompilationCallback_SSE\n" 244193323Sed TYPE_FUNCTION(X86CompilationCallback_SSE) 245193323Sed ASMPREFIX "X86CompilationCallback_SSE:\n" 246193323Sed CFI(".cfi_startproc\n") 247193323Sed "pushl %ebp\n" 248193323Sed CFI(".cfi_def_cfa_offset 8\n") 249193323Sed CFI(".cfi_offset %ebp, -8\n") 250193323Sed "movl %esp, %ebp\n" // Standard prologue 251193323Sed CFI(".cfi_def_cfa_register %ebp\n") 252193323Sed "pushl %eax\n" 253193323Sed CFI(".cfi_rel_offset %eax, 0\n") 254193323Sed "pushl %edx\n" // Save EAX/EDX/ECX 255193323Sed CFI(".cfi_rel_offset %edx, 4\n") 256193323Sed "pushl %ecx\n" 257193323Sed CFI(".cfi_rel_offset %ecx, 8\n") 258193323Sed "andl $-16, %esp\n" // Align ESP on 16-byte boundary 259193323Sed // Save all XMM arg registers 260193323Sed "subl $64, %esp\n" 261193323Sed // FIXME: provide frame move information for xmm registers. 262193323Sed // This can be tricky, because CFA register is ebp (unaligned) 263193323Sed // and we need to produce offsets relative to it. 264193323Sed "movaps %xmm0, (%esp)\n" 265193323Sed "movaps %xmm1, 16(%esp)\n" 266193323Sed "movaps %xmm2, 32(%esp)\n" 267193323Sed "movaps %xmm3, 48(%esp)\n" 268193323Sed "subl $16, %esp\n" 269193323Sed "movl 4(%ebp), %eax\n" // Pass prev frame and return address 270193323Sed "movl %eax, 4(%esp)\n" 271193323Sed "movl %ebp, (%esp)\n" 272252723Sdim "call " ASMPREFIX "LLVMX86CompilationCallback2\n" 273193323Sed "addl $16, %esp\n" 274193323Sed "movaps 48(%esp), %xmm3\n" 275193323Sed CFI(".cfi_restore %xmm3\n") 276193323Sed "movaps 32(%esp), %xmm2\n" 277193323Sed CFI(".cfi_restore %xmm2\n") 278193323Sed "movaps 16(%esp), %xmm1\n" 279193323Sed CFI(".cfi_restore %xmm1\n") 280193323Sed "movaps (%esp), %xmm0\n" 281193323Sed CFI(".cfi_restore %xmm0\n") 282193323Sed "movl %ebp, %esp\n" // Restore ESP 283193323Sed CFI(".cfi_def_cfa_register esp\n") 284193323Sed "subl $12, %esp\n" 285193323Sed CFI(".cfi_adjust_cfa_offset 12\n") 286193323Sed "popl %ecx\n" 287193323Sed CFI(".cfi_adjust_cfa_offset -4\n") 288193323Sed CFI(".cfi_restore %ecx\n") 289193323Sed "popl %edx\n" 290193323Sed CFI(".cfi_adjust_cfa_offset -4\n") 291193323Sed CFI(".cfi_restore %edx\n") 292193323Sed "popl %eax\n" 293193323Sed CFI(".cfi_adjust_cfa_offset -4\n") 294193323Sed CFI(".cfi_restore %eax\n") 295193323Sed "popl %ebp\n" 296193323Sed CFI(".cfi_adjust_cfa_offset -4\n") 297193323Sed CFI(".cfi_restore %ebp\n") 298193323Sed "ret\n" 299193323Sed CFI(".cfi_endproc\n") 300193323Sed SIZE(X86CompilationCallback_SSE) 301193323Sed ); 302193323Sed# else 303252723Sdim void LLVMX86CompilationCallback2(intptr_t *StackPtr, intptr_t RetAddr); 304193323Sed 305193323Sed _declspec(naked) void X86CompilationCallback(void) { 306193323Sed __asm { 307193323Sed push ebp 308193323Sed mov ebp, esp 309193323Sed push eax 310193323Sed push edx 311193323Sed push ecx 312193323Sed and esp, -16 313203954Srdivacky sub esp, 16 314193323Sed mov eax, dword ptr [ebp+4] 315193323Sed mov dword ptr [esp+4], eax 316193323Sed mov dword ptr [esp], ebp 317252723Sdim call LLVMX86CompilationCallback2 318193323Sed mov esp, ebp 319193323Sed sub esp, 12 320193323Sed pop ecx 321193323Sed pop edx 322193323Sed pop eax 323193323Sed pop ebp 324193323Sed ret 325193323Sed } 326193323Sed } 327193323Sed 328193323Sed# endif // _MSC_VER 329193323Sed 330193323Sed#else // Not an i386 host 331193323Sed void X86CompilationCallback() { 332198090Srdivacky llvm_unreachable("Cannot call X86CompilationCallback() on a non-x86 arch!"); 333193323Sed } 334193323Sed#endif 335193323Sed} 336193323Sed 337252723Sdim/// This is the target-specific function invoked by the 338193323Sed/// function stub when we did not know the real target of a call. This function 339193323Sed/// must locate the start of the stub or call site and pass it into the JIT 340193323Sed/// compiler function. 341198090Srdivackyextern "C" { 342263509SdimLLVM_ATTRIBUTE_USED // Referenced from inline asm. 343252723SdimLLVM_LIBRARY_VISIBILITY void LLVMX86CompilationCallback2(intptr_t *StackPtr, 344252723Sdim intptr_t RetAddr) { 345193323Sed intptr_t *RetAddrLoc = &StackPtr[1]; 346252723Sdim // We are reading raw stack data here. Tell MemorySanitizer that it is 347252723Sdim // sufficiently initialized. 348252723Sdim __msan_unpoison(RetAddrLoc, sizeof(*RetAddrLoc)); 349193323Sed assert(*RetAddrLoc == RetAddr && 350193323Sed "Could not find return address on the stack!"); 351193323Sed 352193323Sed // It's a stub if there is an interrupt marker after the call. 353198090Srdivacky bool isStub = ((unsigned char*)RetAddr)[0] == 0xCE; 354193323Sed 355193323Sed // The call instruction should have pushed the return value onto the stack... 356193323Sed#if defined (X86_64_JIT) 357193323Sed RetAddr--; // Backtrack to the reference itself... 358193323Sed#else 359193323Sed RetAddr -= 4; // Backtrack to the reference itself... 360193323Sed#endif 361193323Sed 362193323Sed#if 0 363202375Srdivacky DEBUG(dbgs() << "In callback! Addr=" << (void*)RetAddr 364198090Srdivacky << " ESP=" << (void*)StackPtr 365198090Srdivacky << ": Resolving call to function: " 366198090Srdivacky << TheVM->getFunctionReferencedName((void*)RetAddr) << "\n"); 367193323Sed#endif 368193323Sed 369193323Sed // Sanity check to make sure this really is a call instruction. 370193323Sed#if defined (X86_64_JIT) 371193323Sed assert(((unsigned char*)RetAddr)[-2] == 0x41 &&"Not a call instr!"); 372193323Sed assert(((unsigned char*)RetAddr)[-1] == 0xFF &&"Not a call instr!"); 373193323Sed#else 374193323Sed assert(((unsigned char*)RetAddr)[-1] == 0xE8 &&"Not a call instr!"); 375193323Sed#endif 376193323Sed 377193323Sed intptr_t NewVal = (intptr_t)JITCompilerFunction((void*)RetAddr); 378193323Sed 379193323Sed // Rewrite the call target... so that we don't end up here every time we 380193323Sed // execute the call. 381193323Sed#if defined (X86_64_JIT) 382199481Srdivacky assert(isStub && 383199481Srdivacky "X86-64 doesn't support rewriting non-stub lazy compilation calls:" 384199481Srdivacky " the call instruction varies too much."); 385193323Sed#else 386193323Sed *(intptr_t *)RetAddr = (intptr_t)(NewVal-RetAddr-4); 387193323Sed#endif 388193323Sed 389193323Sed if (isStub) { 390193323Sed // If this is a stub, rewrite the call into an unconditional branch 391193323Sed // instruction so that two return addresses are not pushed onto the stack 392193323Sed // when the requested function finally gets called. This also makes the 393198090Srdivacky // 0xCE byte (interrupt) dead, so the marker doesn't effect anything. 394193323Sed#if defined (X86_64_JIT) 395193323Sed // If the target address is within 32-bit range of the stub, use a 396193323Sed // PC-relative branch instead of loading the actual address. (This is 397193323Sed // considerably shorter than the 64-bit immediate load already there.) 398193323Sed // We assume here intptr_t is 64 bits. 399193323Sed intptr_t diff = NewVal-RetAddr+7; 400193323Sed if (diff >= -2147483648LL && diff <= 2147483647LL) { 401193323Sed *(unsigned char*)(RetAddr-0xc) = 0xE9; 402193323Sed *(intptr_t *)(RetAddr-0xb) = diff & 0xffffffff; 403193323Sed } else { 404193323Sed *(intptr_t *)(RetAddr - 0xa) = NewVal; 405193323Sed ((unsigned char*)RetAddr)[0] = (2 | (4 << 3) | (3 << 6)); 406193323Sed } 407205218Srdivacky sys::ValgrindDiscardTranslations((void*)(RetAddr-0xc), 0xd); 408193323Sed#else 409193323Sed ((unsigned char*)RetAddr)[-1] = 0xE9; 410205218Srdivacky sys::ValgrindDiscardTranslations((void*)(RetAddr-1), 5); 411193323Sed#endif 412193323Sed } 413193323Sed 414193323Sed // Change the return address to reexecute the call instruction... 415193323Sed#if defined (X86_64_JIT) 416193323Sed *RetAddrLoc -= 0xd; 417193323Sed#else 418193323Sed *RetAddrLoc -= 5; 419193323Sed#endif 420193323Sed} 421198090Srdivacky} 422193323Sed 423193323SedTargetJITInfo::LazyResolverFn 424193323SedX86JITInfo::getLazyResolverFunction(JITCompilerFn F) { 425235633Sdim TsanIgnoreWritesBegin(); 426193323Sed JITCompilerFunction = F; 427235633Sdim TsanIgnoreWritesEnd(); 428193323Sed 429193323Sed#if defined (X86_32_JIT) && !defined (_MSC_VER) 430198090Srdivacky if (Subtarget->hasSSE1()) 431198090Srdivacky return X86CompilationCallback_SSE; 432193323Sed#endif 433193323Sed 434193323Sed return X86CompilationCallback; 435193323Sed} 436193323Sed 437198090SrdivackyX86JITInfo::X86JITInfo(X86TargetMachine &tm) : TM(tm) { 438198090Srdivacky Subtarget = &TM.getSubtarget<X86Subtarget>(); 439198090Srdivacky useGOT = 0; 440198090Srdivacky TLSOffset = 0; 441198090Srdivacky} 442198090Srdivacky 443193323Sedvoid *X86JITInfo::emitGlobalValueIndirectSym(const GlobalValue* GV, void *ptr, 444193323Sed JITCodeEmitter &JCE) { 445193323Sed#if defined (X86_64_JIT) 446201360Srdivacky const unsigned Alignment = 8; 447201360Srdivacky uint8_t Buffer[8]; 448201360Srdivacky uint8_t *Cur = Buffer; 449201360Srdivacky MachineCodeEmitter::emitWordLEInto(Cur, (unsigned)(intptr_t)ptr); 450201360Srdivacky MachineCodeEmitter::emitWordLEInto(Cur, (unsigned)(((intptr_t)ptr) >> 32)); 451193323Sed#else 452201360Srdivacky const unsigned Alignment = 4; 453201360Srdivacky uint8_t Buffer[4]; 454201360Srdivacky uint8_t *Cur = Buffer; 455201360Srdivacky MachineCodeEmitter::emitWordLEInto(Cur, (intptr_t)ptr); 456193323Sed#endif 457201360Srdivacky return JCE.allocIndirectGV(GV, Buffer, sizeof(Buffer), Alignment); 458193323Sed} 459193323Sed 460199989SrdivackyTargetJITInfo::StubLayout X86JITInfo::getStubLayout() { 461199989Srdivacky // The 64-bit stub contains: 462199989Srdivacky // movabs r10 <- 8-byte-target-address # 10 bytes 463199989Srdivacky // call|jmp *r10 # 3 bytes 464199989Srdivacky // The 32-bit stub contains a 5-byte call|jmp. 465199989Srdivacky // If the stub is a call to the compilation callback, an extra byte is added 466199989Srdivacky // to mark it as a stub. 467199989Srdivacky StubLayout Result = {14, 4}; 468199989Srdivacky return Result; 469199989Srdivacky} 470199989Srdivacky 471199989Srdivackyvoid *X86JITInfo::emitFunctionStub(const Function* F, void *Target, 472193323Sed JITCodeEmitter &JCE) { 473218893Sdim // Note, we cast to intptr_t here to silence a -pedantic warning that 474193323Sed // complains about casting a function pointer to a normal pointer. 475193323Sed#if defined (X86_32_JIT) && !defined (_MSC_VER) 476199989Srdivacky bool NotCC = (Target != (void*)(intptr_t)X86CompilationCallback && 477199989Srdivacky Target != (void*)(intptr_t)X86CompilationCallback_SSE); 478193323Sed#else 479199989Srdivacky bool NotCC = Target != (void*)(intptr_t)X86CompilationCallback; 480193323Sed#endif 481199989Srdivacky JCE.emitAlignment(4); 482199989Srdivacky void *Result = (void*)JCE.getCurrentPCValue(); 483193323Sed if (NotCC) { 484193323Sed#if defined (X86_64_JIT) 485193323Sed JCE.emitByte(0x49); // REX prefix 486193323Sed JCE.emitByte(0xB8+2); // movabsq r10 487199989Srdivacky JCE.emitWordLE((unsigned)(intptr_t)Target); 488199989Srdivacky JCE.emitWordLE((unsigned)(((intptr_t)Target) >> 32)); 489193323Sed JCE.emitByte(0x41); // REX prefix 490193323Sed JCE.emitByte(0xFF); // jmpq *r10 491193323Sed JCE.emitByte(2 | (4 << 3) | (3 << 6)); 492193323Sed#else 493193323Sed JCE.emitByte(0xE9); 494199989Srdivacky JCE.emitWordLE((intptr_t)Target-JCE.getCurrentPCValue()-4); 495193323Sed#endif 496199989Srdivacky return Result; 497193323Sed } 498193323Sed 499193323Sed#if defined (X86_64_JIT) 500193323Sed JCE.emitByte(0x49); // REX prefix 501193323Sed JCE.emitByte(0xB8+2); // movabsq r10 502199989Srdivacky JCE.emitWordLE((unsigned)(intptr_t)Target); 503199989Srdivacky JCE.emitWordLE((unsigned)(((intptr_t)Target) >> 32)); 504193323Sed JCE.emitByte(0x41); // REX prefix 505193323Sed JCE.emitByte(0xFF); // callq *r10 506193323Sed JCE.emitByte(2 | (2 << 3) | (3 << 6)); 507193323Sed#else 508193323Sed JCE.emitByte(0xE8); // Call with 32 bit pc-rel destination... 509193323Sed 510199989Srdivacky JCE.emitWordLE((intptr_t)Target-JCE.getCurrentPCValue()-4); 511193323Sed#endif 512193323Sed 513198090Srdivacky // This used to use 0xCD, but that value is used by JITMemoryManager to 514198090Srdivacky // initialize the buffer with garbage, which means it may follow a 515252723Sdim // noreturn function call, confusing LLVMX86CompilationCallback2. PR 4929. 516198090Srdivacky JCE.emitByte(0xCE); // Interrupt - Just a marker identifying the stub! 517199989Srdivacky return Result; 518193323Sed} 519193323Sed 520193323Sed/// getPICJumpTableEntry - Returns the value of the jumptable entry for the 521193323Sed/// specific basic block. 522193323Seduintptr_t X86JITInfo::getPICJumpTableEntry(uintptr_t BB, uintptr_t Entry) { 523193323Sed#if defined(X86_64_JIT) 524193323Sed return BB - Entry; 525193323Sed#else 526193323Sed return BB - PICBase; 527193323Sed#endif 528193323Sed} 529193323Sed 530245431Sdimtemplate<typename T> static void addUnaligned(void *Pos, T Delta) { 531245431Sdim T Value; 532245431Sdim std::memcpy(reinterpret_cast<char*>(&Value), reinterpret_cast<char*>(Pos), 533245431Sdim sizeof(T)); 534245431Sdim Value += Delta; 535245431Sdim std::memcpy(reinterpret_cast<char*>(Pos), reinterpret_cast<char*>(&Value), 536245431Sdim sizeof(T)); 537245431Sdim} 538245431Sdim 539193323Sed/// relocate - Before the JIT can run a block of code that has been emitted, 540193323Sed/// it must rewrite the code to contain the actual addresses of any 541193323Sed/// referenced global symbols. 542193323Sedvoid X86JITInfo::relocate(void *Function, MachineRelocation *MR, 543193323Sed unsigned NumRelocs, unsigned char* GOTBase) { 544193323Sed for (unsigned i = 0; i != NumRelocs; ++i, ++MR) { 545193323Sed void *RelocPos = (char*)Function + MR->getMachineCodeOffset(); 546193323Sed intptr_t ResultPtr = (intptr_t)MR->getResultPointer(); 547193323Sed switch ((X86::RelocationType)MR->getRelocationType()) { 548193323Sed case X86::reloc_pcrel_word: { 549193323Sed // PC relative relocation, add the relocated value to the value already in 550193323Sed // memory, after we adjust it for where the PC is. 551193323Sed ResultPtr = ResultPtr -(intptr_t)RelocPos - 4 - MR->getConstantVal(); 552245431Sdim addUnaligned<unsigned>(RelocPos, ResultPtr); 553193323Sed break; 554193323Sed } 555193323Sed case X86::reloc_picrel_word: { 556193323Sed // PIC base relative relocation, add the relocated value to the value 557193323Sed // already in memory, after we adjust it for where the PIC base is. 558193323Sed ResultPtr = ResultPtr - ((intptr_t)Function + MR->getConstantVal()); 559245431Sdim addUnaligned<unsigned>(RelocPos, ResultPtr); 560193323Sed break; 561193323Sed } 562193323Sed case X86::reloc_absolute_word: 563198090Srdivacky case X86::reloc_absolute_word_sext: 564193323Sed // Absolute relocation, just add the relocated value to the value already 565193323Sed // in memory. 566245431Sdim addUnaligned<unsigned>(RelocPos, ResultPtr); 567193323Sed break; 568193323Sed case X86::reloc_absolute_dword: 569245431Sdim addUnaligned<intptr_t>(RelocPos, ResultPtr); 570193323Sed break; 571193323Sed } 572193323Sed } 573193323Sed} 574193323Sed 575193323Sedchar* X86JITInfo::allocateThreadLocalMemory(size_t size) { 576193323Sed#if defined(X86_32_JIT) && !defined(__APPLE__) && !defined(_MSC_VER) 577193323Sed TLSOffset -= size; 578193323Sed return TLSOffset; 579193323Sed#else 580198090Srdivacky llvm_unreachable("Cannot allocate thread local storage on this arch!"); 581193323Sed#endif 582193323Sed} 583