1193323Sed//===-- JITEmitter.cpp - Write machine code to executable memory ----------===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This file defines a MachineCodeEmitter object that is used by the JIT to 11193323Sed// write machine code to memory and remember where relocatable values are. 12193323Sed// 13193323Sed//===----------------------------------------------------------------------===// 14193323Sed 15193323Sed#define DEBUG_TYPE "jit" 16193323Sed#include "JIT.h" 17252723Sdim#include "llvm/ADT/DenseMap.h" 18198090Srdivacky#include "llvm/ADT/OwningPtr.h" 19252723Sdim#include "llvm/ADT/SmallPtrSet.h" 20252723Sdim#include "llvm/ADT/SmallVector.h" 21252723Sdim#include "llvm/ADT/Statistic.h" 22252723Sdim#include "llvm/ADT/ValueMap.h" 23193323Sed#include "llvm/CodeGen/JITCodeEmitter.h" 24207618Srdivacky#include "llvm/CodeGen/MachineCodeInfo.h" 25193323Sed#include "llvm/CodeGen/MachineConstantPool.h" 26252723Sdim#include "llvm/CodeGen/MachineFunction.h" 27193323Sed#include "llvm/CodeGen/MachineJumpTableInfo.h" 28193323Sed#include "llvm/CodeGen/MachineModuleInfo.h" 29193323Sed#include "llvm/CodeGen/MachineRelocation.h" 30252723Sdim#include "llvm/DebugInfo.h" 31195098Sed#include "llvm/ExecutionEngine/GenericValue.h" 32195098Sed#include "llvm/ExecutionEngine/JITEventListener.h" 33193323Sed#include "llvm/ExecutionEngine/JITMemoryManager.h" 34252723Sdim#include "llvm/IR/Constants.h" 35252723Sdim#include "llvm/IR/DataLayout.h" 36252723Sdim#include "llvm/IR/DerivedTypes.h" 37252723Sdim#include "llvm/IR/Module.h" 38193323Sed#include "llvm/Support/Debug.h" 39252723Sdim#include "llvm/Support/Disassembler.h" 40198090Srdivacky#include "llvm/Support/ErrorHandling.h" 41203954Srdivacky#include "llvm/Support/ManagedStatic.h" 42252723Sdim#include "llvm/Support/Memory.h" 43193323Sed#include "llvm/Support/MutexGuard.h" 44193323Sed#include "llvm/Support/ValueHandle.h" 45198090Srdivacky#include "llvm/Support/raw_ostream.h" 46252723Sdim#include "llvm/Target/TargetInstrInfo.h" 47252723Sdim#include "llvm/Target/TargetJITInfo.h" 48252723Sdim#include "llvm/Target/TargetMachine.h" 49252723Sdim#include "llvm/Target/TargetOptions.h" 50193323Sed#include <algorithm> 51193323Sed#ifndef NDEBUG 52193323Sed#include <iomanip> 53193323Sed#endif 54193323Sedusing namespace llvm; 55193323Sed 56193323SedSTATISTIC(NumBytes, "Number of bytes of machine code compiled"); 57193323SedSTATISTIC(NumRelos, "Number of relocations applied"); 58198090SrdivackySTATISTIC(NumRetries, "Number of retries with more memory"); 59193323Sed 60193323Sed 61201360Srdivacky// A declaration may stop being a declaration once it's fully read from bitcode. 62201360Srdivacky// This function returns true if F is fully read and is still a declaration. 63201360Srdivackystatic bool isNonGhostDeclaration(const Function *F) { 64203954Srdivacky return F->isDeclaration() && !F->isMaterializable(); 65201360Srdivacky} 66201360Srdivacky 67193323Sed//===----------------------------------------------------------------------===// 68193323Sed// JIT lazy compilation code. 69193323Sed// 70193323Sednamespace { 71199481Srdivacky class JITEmitter; 72198892Srdivacky class JITResolverState; 73198892Srdivacky 74198892Srdivacky template<typename ValueTy> 75198892Srdivacky struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> { 76198892Srdivacky typedef JITResolverState *ExtraData; 77198892Srdivacky static void onRAUW(JITResolverState *, Value *Old, Value *New) { 78235633Sdim llvm_unreachable("The JIT doesn't know how to handle a" 79235633Sdim " RAUW on a value it has emitted."); 80198892Srdivacky } 81198892Srdivacky }; 82198892Srdivacky 83198892Srdivacky struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> { 84198892Srdivacky typedef JITResolverState *ExtraData; 85198892Srdivacky static void onDelete(JITResolverState *JRS, Function *F); 86198892Srdivacky }; 87198892Srdivacky 88193323Sed class JITResolverState { 89193323Sed public: 90198892Srdivacky typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> > 91199989Srdivacky FunctionToLazyStubMapTy; 92198396Srdivacky typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy; 93198892Srdivacky typedef ValueMap<Function *, SmallPtrSet<void*, 1>, 94198892Srdivacky CallSiteValueMapConfig> FunctionToCallSitesMapTy; 95193323Sed typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy; 96193323Sed private: 97199989Srdivacky /// FunctionToLazyStubMap - Keep track of the lazy stub created for a 98199989Srdivacky /// particular function so that we can reuse them if necessary. 99199989Srdivacky FunctionToLazyStubMapTy FunctionToLazyStubMap; 100193323Sed 101198396Srdivacky /// CallSiteToFunctionMap - Keep track of the function that each lazy call 102198396Srdivacky /// site corresponds to, and vice versa. 103198396Srdivacky CallSiteToFunctionMapTy CallSiteToFunctionMap; 104198396Srdivacky FunctionToCallSitesMapTy FunctionToCallSitesMap; 105193323Sed 106193323Sed /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a 107193323Sed /// particular GlobalVariable so that we can reuse them if necessary. 108193323Sed GlobalToIndirectSymMapTy GlobalToIndirectSymMap; 109193323Sed 110245431Sdim#ifndef NDEBUG 111203954Srdivacky /// Instance of the JIT this ResolverState serves. 112203954Srdivacky JIT *TheJIT; 113245431Sdim#endif 114203954Srdivacky 115193323Sed public: 116203954Srdivacky JITResolverState(JIT *jit) : FunctionToLazyStubMap(this), 117245431Sdim FunctionToCallSitesMap(this) { 118245431Sdim#ifndef NDEBUG 119245431Sdim TheJIT = jit; 120245431Sdim#endif 121245431Sdim } 122198892Srdivacky 123199989Srdivacky FunctionToLazyStubMapTy& getFunctionToLazyStubMap( 124199989Srdivacky const MutexGuard& locked) { 125193323Sed assert(locked.holds(TheJIT->lock)); 126199989Srdivacky return FunctionToLazyStubMap; 127193323Sed } 128193323Sed 129221345Sdim GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap(const MutexGuard& lck) { 130221345Sdim assert(lck.holds(TheJIT->lock)); 131198396Srdivacky return GlobalToIndirectSymMap; 132193323Sed } 133193323Sed 134221345Sdim std::pair<void *, Function *> LookupFunctionFromCallSite( 135198396Srdivacky const MutexGuard &locked, void *CallSite) const { 136193323Sed assert(locked.holds(TheJIT->lock)); 137198396Srdivacky 138221345Sdim // The address given to us for the stub may not be exactly right, it 139221345Sdim // might be a little bit after the stub. As such, use upper_bound to 140221345Sdim // find it. 141198396Srdivacky CallSiteToFunctionMapTy::const_iterator I = 142198396Srdivacky CallSiteToFunctionMap.upper_bound(CallSite); 143198396Srdivacky assert(I != CallSiteToFunctionMap.begin() && 144198396Srdivacky "This is not a known call site!"); 145198396Srdivacky --I; 146198396Srdivacky return *I; 147193323Sed } 148198396Srdivacky 149198396Srdivacky void AddCallSite(const MutexGuard &locked, void *CallSite, Function *F) { 150198396Srdivacky assert(locked.holds(TheJIT->lock)); 151198396Srdivacky 152198892Srdivacky bool Inserted = CallSiteToFunctionMap.insert( 153198892Srdivacky std::make_pair(CallSite, F)).second; 154198892Srdivacky (void)Inserted; 155198892Srdivacky assert(Inserted && "Pair was already in CallSiteToFunctionMap"); 156198396Srdivacky FunctionToCallSitesMap[F].insert(CallSite); 157198396Srdivacky } 158198396Srdivacky 159204792Srdivacky void EraseAllCallSitesForPrelocked(Function *F); 160198396Srdivacky 161204792Srdivacky // Erases _all_ call sites regardless of their function. This is used to 162204792Srdivacky // unregister the stub addresses from the StubToResolverMap in 163204792Srdivacky // ~JITResolver(). 164204792Srdivacky void EraseAllCallSitesPrelocked(); 165193323Sed }; 166193323Sed 167193323Sed /// JITResolver - Keep track of, and resolve, call sites for functions that 168193323Sed /// have not yet been compiled. 169193323Sed class JITResolver { 170199989Srdivacky typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy; 171198396Srdivacky typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy; 172193323Sed typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; 173193323Sed 174193323Sed /// LazyResolverFn - The target lazy resolver function that we actually 175193323Sed /// rewrite instructions to use. 176193323Sed TargetJITInfo::LazyResolverFn LazyResolverFn; 177193323Sed 178193323Sed JITResolverState state; 179193323Sed 180199989Srdivacky /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap 181199989Srdivacky /// for external functions. TODO: Of course, external functions don't need 182199989Srdivacky /// a lazy stub. It's actually here to make it more likely that far calls 183199989Srdivacky /// succeed, but no single stub can guarantee that. I'll remove this in a 184199989Srdivacky /// subsequent checkin when I actually fix far calls. 185193323Sed std::map<void*, void*> ExternalFnToStubMap; 186193323Sed 187193323Sed /// revGOTMap - map addresses to indexes in the GOT 188193323Sed std::map<void*, unsigned> revGOTMap; 189193323Sed unsigned nextGOTIndex; 190193323Sed 191199481Srdivacky JITEmitter &JE; 192199481Srdivacky 193203954Srdivacky /// Instance of JIT corresponding to this Resolver. 194203954Srdivacky JIT *TheJIT; 195203954Srdivacky 196193323Sed public: 197203954Srdivacky explicit JITResolver(JIT &jit, JITEmitter &je) 198203954Srdivacky : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) { 199193323Sed LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); 200193323Sed } 201199481Srdivacky 202204792Srdivacky ~JITResolver(); 203204792Srdivacky 204199989Srdivacky /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's 205199989Srdivacky /// lazy-compilation stub if it has already been created. 206199989Srdivacky void *getLazyFunctionStubIfAvailable(Function *F); 207193323Sed 208199989Srdivacky /// getLazyFunctionStub - This returns a pointer to a function's 209199989Srdivacky /// lazy-compilation stub, creating one on demand as needed. 210199989Srdivacky void *getLazyFunctionStub(Function *F); 211193323Sed 212193323Sed /// getExternalFunctionStub - Return a stub for the function at the 213193323Sed /// specified address, created lazily on demand. 214193323Sed void *getExternalFunctionStub(void *FnAddr); 215193323Sed 216193323Sed /// getGlobalValueIndirectSym - Return an indirect symbol containing the 217193323Sed /// specified GV address. 218193323Sed void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); 219193323Sed 220193323Sed /// getGOTIndexForAddress - Return a new or existing index in the GOT for 221193323Sed /// an address. This function only manages slots, it does not manage the 222193323Sed /// contents of the slots or the memory associated with the GOT. 223193323Sed unsigned getGOTIndexForAddr(void *addr); 224193323Sed 225193323Sed /// JITCompilerFn - This function is called to resolve a stub to a compiled 226193323Sed /// address. If the LLVM Function corresponding to the stub has not yet 227193323Sed /// been compiled, this function compiles it first. 228193323Sed static void *JITCompilerFn(void *Stub); 229193323Sed }; 230199481Srdivacky 231203954Srdivacky class StubToResolverMapTy { 232203954Srdivacky /// Map a stub address to a specific instance of a JITResolver so that 233203954Srdivacky /// lazily-compiled functions can find the right resolver to use. 234203954Srdivacky /// 235203954Srdivacky /// Guarded by Lock. 236203954Srdivacky std::map<void*, JITResolver*> Map; 237203954Srdivacky 238203954Srdivacky /// Guards Map from concurrent accesses. 239203954Srdivacky mutable sys::Mutex Lock; 240203954Srdivacky 241203954Srdivacky public: 242203954Srdivacky /// Registers a Stub to be resolved by Resolver. 243203954Srdivacky void RegisterStubResolver(void *Stub, JITResolver *Resolver) { 244203954Srdivacky MutexGuard guard(Lock); 245203954Srdivacky Map.insert(std::make_pair(Stub, Resolver)); 246203954Srdivacky } 247203954Srdivacky /// Unregisters the Stub when it's invalidated. 248203954Srdivacky void UnregisterStubResolver(void *Stub) { 249203954Srdivacky MutexGuard guard(Lock); 250203954Srdivacky Map.erase(Stub); 251203954Srdivacky } 252203954Srdivacky /// Returns the JITResolver instance that owns the Stub. 253203954Srdivacky JITResolver *getResolverFromStub(void *Stub) const { 254203954Srdivacky MutexGuard guard(Lock); 255203954Srdivacky // The address given to us for the stub may not be exactly right, it might 256203954Srdivacky // be a little bit after the stub. As such, use upper_bound to find it. 257203954Srdivacky // This is the same trick as in LookupFunctionFromCallSite from 258203954Srdivacky // JITResolverState. 259203954Srdivacky std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub); 260203954Srdivacky assert(I != Map.begin() && "This is not a known stub!"); 261203954Srdivacky --I; 262203954Srdivacky return I->second; 263203954Srdivacky } 264204792Srdivacky /// True if any stubs refer to the given resolver. Only used in an assert(). 265204792Srdivacky /// O(N) 266204792Srdivacky bool ResolverHasStubs(JITResolver* Resolver) const { 267204792Srdivacky MutexGuard guard(Lock); 268204792Srdivacky for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(), 269204792Srdivacky E = Map.end(); I != E; ++I) { 270204792Srdivacky if (I->second == Resolver) 271204792Srdivacky return true; 272204792Srdivacky } 273204792Srdivacky return false; 274204792Srdivacky } 275203954Srdivacky }; 276203954Srdivacky /// This needs to be static so that a lazy call stub can access it with no 277203954Srdivacky /// context except the address of the stub. 278203954Srdivacky ManagedStatic<StubToResolverMapTy> StubToResolverMap; 279203954Srdivacky 280199481Srdivacky /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is 281199481Srdivacky /// used to output functions to memory for execution. 282199481Srdivacky class JITEmitter : public JITCodeEmitter { 283199481Srdivacky JITMemoryManager *MemMgr; 284199481Srdivacky 285201360Srdivacky // When outputting a function stub in the context of some other function, we 286201360Srdivacky // save BufferBegin/BufferEnd/CurBufferPtr here. 287201360Srdivacky uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; 288201360Srdivacky 289199481Srdivacky // When reattempting to JIT a function after running out of space, we store 290199481Srdivacky // the estimated size of the function we're trying to JIT here, so we can 291199481Srdivacky // ask the memory manager for at least this much space. When we 292199481Srdivacky // successfully emit the function, we reset this back to zero. 293199481Srdivacky uintptr_t SizeEstimate; 294199481Srdivacky 295199481Srdivacky /// Relocations - These are the relocations that the function needs, as 296199481Srdivacky /// emitted. 297199481Srdivacky std::vector<MachineRelocation> Relocations; 298199481Srdivacky 299199481Srdivacky /// MBBLocations - This vector is a mapping from MBB ID's to their address. 300199481Srdivacky /// It is filled in by the StartMachineBasicBlock callback and queried by 301199481Srdivacky /// the getMachineBasicBlockAddress callback. 302199481Srdivacky std::vector<uintptr_t> MBBLocations; 303199481Srdivacky 304199481Srdivacky /// ConstantPool - The constant pool for the current function. 305199481Srdivacky /// 306199481Srdivacky MachineConstantPool *ConstantPool; 307199481Srdivacky 308199481Srdivacky /// ConstantPoolBase - A pointer to the first entry in the constant pool. 309199481Srdivacky /// 310199481Srdivacky void *ConstantPoolBase; 311199481Srdivacky 312199481Srdivacky /// ConstPoolAddresses - Addresses of individual constant pool entries. 313199481Srdivacky /// 314199481Srdivacky SmallVector<uintptr_t, 8> ConstPoolAddresses; 315199481Srdivacky 316199481Srdivacky /// JumpTable - The jump tables for the current function. 317199481Srdivacky /// 318199481Srdivacky MachineJumpTableInfo *JumpTable; 319199481Srdivacky 320199481Srdivacky /// JumpTableBase - A pointer to the first entry in the jump table. 321199481Srdivacky /// 322199481Srdivacky void *JumpTableBase; 323199481Srdivacky 324199481Srdivacky /// Resolver - This contains info about the currently resolved functions. 325199481Srdivacky JITResolver Resolver; 326199481Srdivacky 327199481Srdivacky /// LabelLocations - This vector is a mapping from Label ID's to their 328199481Srdivacky /// address. 329205218Srdivacky DenseMap<MCSymbol*, uintptr_t> LabelLocations; 330199481Srdivacky 331199481Srdivacky /// MMI - Machine module info for exception informations 332199481Srdivacky MachineModuleInfo* MMI; 333199481Srdivacky 334199481Srdivacky // CurFn - The llvm function being emitted. Only valid during 335199481Srdivacky // finishFunction(). 336199481Srdivacky const Function *CurFn; 337199481Srdivacky 338199481Srdivacky /// Information about emitted code, which is passed to the 339199481Srdivacky /// JITEventListeners. This is reset in startFunction and used in 340199481Srdivacky /// finishFunction. 341199481Srdivacky JITEvent_EmittedFunctionDetails EmissionDetails; 342199481Srdivacky 343199481Srdivacky struct EmittedCode { 344199481Srdivacky void *FunctionBody; // Beginning of the function's allocation. 345199481Srdivacky void *Code; // The address the function's code actually starts at. 346199481Srdivacky void *ExceptionTable; 347199481Srdivacky EmittedCode() : FunctionBody(0), Code(0), ExceptionTable(0) {} 348199481Srdivacky }; 349199481Srdivacky struct EmittedFunctionConfig : public ValueMapConfig<const Function*> { 350199481Srdivacky typedef JITEmitter *ExtraData; 351199481Srdivacky static void onDelete(JITEmitter *, const Function*); 352199481Srdivacky static void onRAUW(JITEmitter *, const Function*, const Function*); 353199481Srdivacky }; 354199481Srdivacky ValueMap<const Function *, EmittedCode, 355199481Srdivacky EmittedFunctionConfig> EmittedFunctions; 356199481Srdivacky 357207618Srdivacky DebugLoc PrevDL; 358199481Srdivacky 359203954Srdivacky /// Instance of the JIT 360203954Srdivacky JIT *TheJIT; 361203954Srdivacky 362199481Srdivacky public: 363199481Srdivacky JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM) 364199481Srdivacky : SizeEstimate(0), Resolver(jit, *this), MMI(0), CurFn(0), 365263509Sdim EmittedFunctions(this), TheJIT(&jit) { 366199481Srdivacky MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); 367199481Srdivacky if (jit.getJITInfo().needsGOT()) { 368199481Srdivacky MemMgr->AllocateGOT(); 369202375Srdivacky DEBUG(dbgs() << "JIT is managing a GOT\n"); 370199481Srdivacky } 371199481Srdivacky 372199481Srdivacky } 373199481Srdivacky ~JITEmitter() { 374199481Srdivacky delete MemMgr; 375199481Srdivacky } 376199481Srdivacky 377199481Srdivacky JITResolver &getJITResolver() { return Resolver; } 378199481Srdivacky 379199481Srdivacky virtual void startFunction(MachineFunction &F); 380199481Srdivacky virtual bool finishFunction(MachineFunction &F); 381199481Srdivacky 382199481Srdivacky void emitConstantPool(MachineConstantPool *MCP); 383199481Srdivacky void initJumpTableInfo(MachineJumpTableInfo *MJTI); 384199481Srdivacky void emitJumpTableInfo(MachineJumpTableInfo *MJTI); 385199481Srdivacky 386201360Srdivacky void startGVStub(const GlobalValue* GV, 387201360Srdivacky unsigned StubSize, unsigned Alignment = 1); 388201360Srdivacky void startGVStub(void *Buffer, unsigned StubSize); 389201360Srdivacky void finishGVStub(); 390201360Srdivacky virtual void *allocIndirectGV(const GlobalValue *GV, 391201360Srdivacky const uint8_t *Buffer, size_t Size, 392201360Srdivacky unsigned Alignment); 393199481Srdivacky 394199481Srdivacky /// allocateSpace - Reserves space in the current block if any, or 395199481Srdivacky /// allocate a new one of the given size. 396199481Srdivacky virtual void *allocateSpace(uintptr_t Size, unsigned Alignment); 397199481Srdivacky 398199481Srdivacky /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, 399199481Srdivacky /// this method does not allocate memory in the current output buffer, 400199481Srdivacky /// because a global may live longer than the current function. 401199481Srdivacky virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment); 402199481Srdivacky 403199481Srdivacky virtual void addRelocation(const MachineRelocation &MR) { 404199481Srdivacky Relocations.push_back(MR); 405199481Srdivacky } 406199481Srdivacky 407199481Srdivacky virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { 408199481Srdivacky if (MBBLocations.size() <= (unsigned)MBB->getNumber()) 409199481Srdivacky MBBLocations.resize((MBB->getNumber()+1)*2); 410199481Srdivacky MBBLocations[MBB->getNumber()] = getCurrentPCValue(); 411210299Sed if (MBB->hasAddressTaken()) 412210299Sed TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(), 413210299Sed (void*)getCurrentPCValue()); 414202375Srdivacky DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" 415199481Srdivacky << (void*) getCurrentPCValue() << "]\n"); 416199481Srdivacky } 417199481Srdivacky 418199481Srdivacky virtual uintptr_t getConstantPoolEntryAddress(unsigned Entry) const; 419199481Srdivacky virtual uintptr_t getJumpTableEntryAddress(unsigned Entry) const; 420199481Srdivacky 421210299Sed virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const{ 422199481Srdivacky assert(MBBLocations.size() > (unsigned)MBB->getNumber() && 423199481Srdivacky MBBLocations[MBB->getNumber()] && "MBB not emitted!"); 424199481Srdivacky return MBBLocations[MBB->getNumber()]; 425199481Srdivacky } 426199481Srdivacky 427199481Srdivacky /// retryWithMoreMemory - Log a retry and deallocate all memory for the 428199481Srdivacky /// given function. Increase the minimum allocation size so that we get 429199481Srdivacky /// more memory next time. 430199481Srdivacky void retryWithMoreMemory(MachineFunction &F); 431199481Srdivacky 432199481Srdivacky /// deallocateMemForFunction - Deallocate all memory for the specified 433199481Srdivacky /// function body. 434199481Srdivacky void deallocateMemForFunction(const Function *F); 435199481Srdivacky 436199481Srdivacky virtual void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn); 437199481Srdivacky 438205218Srdivacky virtual void emitLabel(MCSymbol *Label) { 439205218Srdivacky LabelLocations[Label] = getCurrentPCValue(); 440199481Srdivacky } 441199481Srdivacky 442207618Srdivacky virtual DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() { 443207618Srdivacky return &LabelLocations; 444207618Srdivacky } 445207618Srdivacky 446205218Srdivacky virtual uintptr_t getLabelAddress(MCSymbol *Label) const { 447205218Srdivacky assert(LabelLocations.count(Label) && "Label not emitted!"); 448205218Srdivacky return LabelLocations.find(Label)->second; 449199481Srdivacky } 450199481Srdivacky 451199481Srdivacky virtual void setModuleInfo(MachineModuleInfo* Info) { 452199481Srdivacky MMI = Info; 453199481Srdivacky } 454199481Srdivacky 455199481Srdivacky private: 456199481Srdivacky void *getPointerToGlobal(GlobalValue *GV, void *Reference, 457199481Srdivacky bool MayNeedFarStub); 458199481Srdivacky void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference); 459199481Srdivacky }; 460193323Sed} 461193323Sed 462198892Srdivackyvoid CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) { 463204792Srdivacky JRS->EraseAllCallSitesForPrelocked(F); 464198892Srdivacky} 465198892Srdivacky 466204792Srdivackyvoid JITResolverState::EraseAllCallSitesForPrelocked(Function *F) { 467204792Srdivacky FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); 468204792Srdivacky if (F2C == FunctionToCallSitesMap.end()) 469204792Srdivacky return; 470204792Srdivacky StubToResolverMapTy &S2RMap = *StubToResolverMap; 471204792Srdivacky for (SmallPtrSet<void*, 1>::const_iterator I = F2C->second.begin(), 472204792Srdivacky E = F2C->second.end(); I != E; ++I) { 473204792Srdivacky S2RMap.UnregisterStubResolver(*I); 474204792Srdivacky bool Erased = CallSiteToFunctionMap.erase(*I); 475204792Srdivacky (void)Erased; 476204792Srdivacky assert(Erased && "Missing call site->function mapping"); 477204792Srdivacky } 478204792Srdivacky FunctionToCallSitesMap.erase(F2C); 479204792Srdivacky} 480204792Srdivacky 481204792Srdivackyvoid JITResolverState::EraseAllCallSitesPrelocked() { 482204792Srdivacky StubToResolverMapTy &S2RMap = *StubToResolverMap; 483204792Srdivacky for (CallSiteToFunctionMapTy::const_iterator 484204792Srdivacky I = CallSiteToFunctionMap.begin(), 485204792Srdivacky E = CallSiteToFunctionMap.end(); I != E; ++I) { 486204792Srdivacky S2RMap.UnregisterStubResolver(I->first); 487204792Srdivacky } 488204792Srdivacky CallSiteToFunctionMap.clear(); 489204792Srdivacky FunctionToCallSitesMap.clear(); 490204792Srdivacky} 491204792Srdivacky 492204792SrdivackyJITResolver::~JITResolver() { 493204792Srdivacky // No need to lock because we're in the destructor, and state isn't shared. 494204792Srdivacky state.EraseAllCallSitesPrelocked(); 495204792Srdivacky assert(!StubToResolverMap->ResolverHasStubs(this) && 496204792Srdivacky "Resolver destroyed with stubs still alive."); 497204792Srdivacky} 498204792Srdivacky 499199989Srdivacky/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub 500193323Sed/// if it has already been created. 501199989Srdivackyvoid *JITResolver::getLazyFunctionStubIfAvailable(Function *F) { 502193323Sed MutexGuard locked(TheJIT->lock); 503193323Sed 504193323Sed // If we already have a stub for this function, recycle it. 505199989Srdivacky return state.getFunctionToLazyStubMap(locked).lookup(F); 506193323Sed} 507193323Sed 508193323Sed/// getFunctionStub - This returns a pointer to a function stub, creating 509193323Sed/// one on demand as needed. 510199989Srdivackyvoid *JITResolver::getLazyFunctionStub(Function *F) { 511193323Sed MutexGuard locked(TheJIT->lock); 512193323Sed 513199989Srdivacky // If we already have a lazy stub for this function, recycle it. 514199989Srdivacky void *&Stub = state.getFunctionToLazyStubMap(locked)[F]; 515193323Sed if (Stub) return Stub; 516193323Sed 517198892Srdivacky // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we 518198892Srdivacky // must resolve the symbol now. 519198892Srdivacky void *Actual = TheJIT->isCompilingLazily() 520198892Srdivacky ? (void *)(intptr_t)LazyResolverFn : (void *)0; 521198892Srdivacky 522193323Sed // If this is an external declaration, attempt to resolve the address now 523193323Sed // to place in the stub. 524201360Srdivacky if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) { 525193323Sed Actual = TheJIT->getPointerToFunction(F); 526193323Sed 527193323Sed // If we resolved the symbol to a null address (eg. a weak external) 528199481Srdivacky // don't emit a stub. Return a null pointer to the application. 529199481Srdivacky if (!Actual) return 0; 530193323Sed } 531193323Sed 532199989Srdivacky TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); 533201360Srdivacky JE.startGVStub(F, SL.Size, SL.Alignment); 534193323Sed // Codegen a new stub, calling the lazy resolver or the actual address of the 535193323Sed // external function, if it was resolved. 536199481Srdivacky Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE); 537201360Srdivacky JE.finishGVStub(); 538193323Sed 539193323Sed if (Actual != (void*)(intptr_t)LazyResolverFn) { 540193323Sed // If we are getting the stub for an external function, we really want the 541193323Sed // address of the stub in the GlobalAddressMap for the JIT, not the address 542193323Sed // of the external function. 543193323Sed TheJIT->updateGlobalMapping(F, Stub); 544193323Sed } 545193323Sed 546202375Srdivacky DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '" 547198090Srdivacky << F->getName() << "'\n"); 548193323Sed 549204792Srdivacky if (TheJIT->isCompilingLazily()) { 550204792Srdivacky // Register this JITResolver as the one corresponding to this call site so 551204792Srdivacky // JITCompilerFn will be able to find it. 552204792Srdivacky StubToResolverMap->RegisterStubResolver(Stub, this); 553203954Srdivacky 554204792Srdivacky // Finally, keep track of the stub-to-Function mapping so that the 555204792Srdivacky // JITCompilerFn knows which function to compile! 556204792Srdivacky state.AddCallSite(locked, Stub, F); 557204792Srdivacky } else if (!Actual) { 558204792Srdivacky // If we are JIT'ing non-lazily but need to call a function that does not 559204792Srdivacky // exist yet, add it to the JIT's work list so that we can fill in the 560204792Srdivacky // stub address later. 561204792Srdivacky assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() && 562204792Srdivacky "'Actual' should have been set above."); 563204792Srdivacky TheJIT->addPendingFunction(F); 564204792Srdivacky } 565198090Srdivacky 566193323Sed return Stub; 567193323Sed} 568193323Sed 569193323Sed/// getGlobalValueIndirectSym - Return a lazy pointer containing the specified 570193323Sed/// GV address. 571193323Sedvoid *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { 572193323Sed MutexGuard locked(TheJIT->lock); 573193323Sed 574193323Sed // If we already have a stub for this global variable, recycle it. 575193323Sed void *&IndirectSym = state.getGlobalToIndirectSymMap(locked)[GV]; 576193323Sed if (IndirectSym) return IndirectSym; 577193323Sed 578193323Sed // Otherwise, codegen a new indirect symbol. 579193323Sed IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, 580199481Srdivacky JE); 581193323Sed 582202375Srdivacky DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym 583198090Srdivacky << "] for GV '" << GV->getName() << "'\n"); 584193323Sed 585193323Sed return IndirectSym; 586193323Sed} 587193323Sed 588193323Sed/// getExternalFunctionStub - Return a stub for the function at the 589193323Sed/// specified address, created lazily on demand. 590193323Sedvoid *JITResolver::getExternalFunctionStub(void *FnAddr) { 591193323Sed // If we already have a stub for this function, recycle it. 592193323Sed void *&Stub = ExternalFnToStubMap[FnAddr]; 593193323Sed if (Stub) return Stub; 594193323Sed 595199989Srdivacky TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); 596201360Srdivacky JE.startGVStub(0, SL.Size, SL.Alignment); 597199481Srdivacky Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr, JE); 598201360Srdivacky JE.finishGVStub(); 599193323Sed 600202375Srdivacky DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub 601198090Srdivacky << "] for external function at '" << FnAddr << "'\n"); 602193323Sed return Stub; 603193323Sed} 604193323Sed 605193323Sedunsigned JITResolver::getGOTIndexForAddr(void* addr) { 606193323Sed unsigned idx = revGOTMap[addr]; 607193323Sed if (!idx) { 608193323Sed idx = ++nextGOTIndex; 609193323Sed revGOTMap[addr] = idx; 610202375Srdivacky DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr [" 611198090Srdivacky << addr << "]\n"); 612193323Sed } 613193323Sed return idx; 614193323Sed} 615193323Sed 616193323Sed/// JITCompilerFn - This function is called when a lazy compilation stub has 617193323Sed/// been entered. It looks up which function this stub corresponds to, compiles 618193323Sed/// it if necessary, then returns the resultant function pointer. 619193323Sedvoid *JITResolver::JITCompilerFn(void *Stub) { 620203954Srdivacky JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub); 621203954Srdivacky assert(JR && "Unable to find the corresponding JITResolver to the call site"); 622199481Srdivacky 623193323Sed Function* F = 0; 624193323Sed void* ActualPtr = 0; 625193323Sed 626193323Sed { 627193323Sed // Only lock for getting the Function. The call getPointerToFunction made 628193323Sed // in this function might trigger function materializing, which requires 629193323Sed // JIT lock to be unlocked. 630203954Srdivacky MutexGuard locked(JR->TheJIT->lock); 631193323Sed 632198396Srdivacky // The address given to us for the stub may not be exactly right, it might 633198396Srdivacky // be a little bit after the stub. As such, use upper_bound to find it. 634221345Sdim std::pair<void*, Function*> I = 635203954Srdivacky JR->state.LookupFunctionFromCallSite(locked, Stub); 636198396Srdivacky F = I.second; 637198396Srdivacky ActualPtr = I.first; 638193323Sed } 639193323Sed 640193323Sed // If we have already code generated the function, just return the address. 641203954Srdivacky void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F); 642199481Srdivacky 643193323Sed if (!Result) { 644193323Sed // Otherwise we don't have it, do lazy compilation now. 645199481Srdivacky 646193323Sed // If lazy compilation is disabled, emit a useful error message and abort. 647203954Srdivacky if (!JR->TheJIT->isCompilingLazily()) { 648221345Sdim report_fatal_error("LLVM JIT requested to do lazy compilation of" 649221345Sdim " function '" 650198090Srdivacky + F->getName() + "' when lazy compiles are disabled!"); 651193323Sed } 652199481Srdivacky 653202375Srdivacky DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName() 654198090Srdivacky << "' In stub ptr = " << Stub << " actual ptr = " 655198090Srdivacky << ActualPtr << "\n"); 656226890Sdim (void)ActualPtr; 657193323Sed 658203954Srdivacky Result = JR->TheJIT->getPointerToFunction(F); 659193323Sed } 660198396Srdivacky 661198396Srdivacky // Reacquire the lock to update the GOT map. 662203954Srdivacky MutexGuard locked(JR->TheJIT->lock); 663193323Sed 664198396Srdivacky // We might like to remove the call site from the CallSiteToFunction map, but 665198396Srdivacky // we can't do that! Multiple threads could be stuck, waiting to acquire the 666198396Srdivacky // lock above. As soon as the 1st function finishes compiling the function, 667198396Srdivacky // the next one will be released, and needs to be able to find the function it 668198396Srdivacky // needs to call. 669193323Sed 670193323Sed // FIXME: We could rewrite all references to this stub if we knew them. 671193323Sed 672193323Sed // What we will do is set the compiled function address to map to the 673193323Sed // same GOT entry as the stub so that later clients may update the GOT 674193323Sed // if they see it still using the stub address. 675193323Sed // Note: this is done so the Resolver doesn't have to manage GOT memory 676193323Sed // Do this without allocating map space if the target isn't using a GOT 677203954Srdivacky if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end()) 678203954Srdivacky JR->revGOTMap[Result] = JR->revGOTMap[Stub]; 679193323Sed 680193323Sed return Result; 681193323Sed} 682193323Sed 683193323Sed//===----------------------------------------------------------------------===// 684193323Sed// JITEmitter code. 685193323Sed// 686193323Sedvoid *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, 687199481Srdivacky bool MayNeedFarStub) { 688193323Sed if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) 689193323Sed return TheJIT->getOrEmitGlobalVariable(GV); 690193323Sed 691193323Sed if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 692193323Sed return TheJIT->getPointerToGlobal(GA->resolveAliasedGlobal(false)); 693193323Sed 694193323Sed // If we have already compiled the function, return a pointer to its body. 695193323Sed Function *F = cast<Function>(V); 696199481Srdivacky 697199989Srdivacky void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F); 698199481Srdivacky if (FnStub) { 699199989Srdivacky // Return the function stub if it's already created. We do this first so 700199989Srdivacky // that we're returning the same address for the function as any previous 701199989Srdivacky // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be 702199989Srdivacky // close enough to call. 703199481Srdivacky return FnStub; 704193323Sed } 705199481Srdivacky 706199989Srdivacky // If we know the target can handle arbitrary-distance calls, try to 707199989Srdivacky // return a direct pointer. 708199989Srdivacky if (!MayNeedFarStub) { 709199989Srdivacky // If we have code, go ahead and return that. 710199989Srdivacky void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); 711199989Srdivacky if (ResultPtr) return ResultPtr; 712193323Sed 713199989Srdivacky // If this is an external function pointer, we can force the JIT to 714199989Srdivacky // 'compile' it, which really just adds it to the map. 715201360Srdivacky if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) 716199989Srdivacky return TheJIT->getPointerToFunction(F); 717199989Srdivacky } 718193323Sed 719204792Srdivacky // Otherwise, we may need a to emit a stub, and, conservatively, we always do 720204792Srdivacky // so. Note that it's possible to return null from getLazyFunctionStub in the 721204792Srdivacky // case of a weak extern that fails to resolve. 722204792Srdivacky return Resolver.getLazyFunctionStub(F); 723193323Sed} 724193323Sed 725199481Srdivackyvoid *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { 726193323Sed // Make sure GV is emitted first, and create a stub containing the fully 727193323Sed // resolved address. 728199481Srdivacky void *GVAddress = getPointerToGlobal(V, Reference, false); 729193323Sed void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); 730193323Sed return StubAddr; 731193323Sed} 732193323Sed 733198090Srdivackyvoid JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { 734206124Srdivacky if (DL.isUnknown()) return; 735206124Srdivacky if (!BeforePrintingInsn) return; 736221345Sdim 737212904Sdim const LLVMContext &Context = EmissionDetails.MF->getFunction()->getContext(); 738198090Srdivacky 739207618Srdivacky if (DL.getScope(Context) != 0 && PrevDL != DL) { 740206124Srdivacky JITEvent_EmittedFunctionDetails::LineStart NextLine; 741206124Srdivacky NextLine.Address = getCurrentPCValue(); 742206124Srdivacky NextLine.Loc = DL; 743206124Srdivacky EmissionDetails.LineStarts.push_back(NextLine); 744206124Srdivacky } 745199481Srdivacky 746207618Srdivacky PrevDL = DL; 747198090Srdivacky} 748198090Srdivacky 749193323Sedstatic unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, 750245431Sdim const DataLayout *TD) { 751193323Sed const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 752193323Sed if (Constants.empty()) return 0; 753193323Sed 754193323Sed unsigned Size = 0; 755193323Sed for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 756193323Sed MachineConstantPoolEntry CPE = Constants[i]; 757193323Sed unsigned AlignMask = CPE.getAlignment() - 1; 758193323Sed Size = (Size + AlignMask) & ~AlignMask; 759226890Sdim Type *Ty = CPE.getType(); 760193323Sed Size += TD->getTypeAllocSize(Ty); 761193323Sed } 762193323Sed return Size; 763193323Sed} 764193323Sed 765193323Sedvoid JITEmitter::startFunction(MachineFunction &F) { 766202375Srdivacky DEBUG(dbgs() << "JIT: Starting CodeGen of Function " 767245431Sdim << F.getName() << "\n"); 768193323Sed 769193323Sed uintptr_t ActualSize = 0; 770193323Sed // Set the memory writable, if it's not already 771193323Sed MemMgr->setMemoryWritable(); 772221345Sdim 773212904Sdim if (SizeEstimate > 0) { 774198090Srdivacky // SizeEstimate will be non-zero on reallocation attempts. 775198090Srdivacky ActualSize = SizeEstimate; 776193323Sed } 777193323Sed 778193323Sed BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), 779193323Sed ActualSize); 780193323Sed BufferEnd = BufferBegin+ActualSize; 781198396Srdivacky EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin; 782198396Srdivacky 783193323Sed // Ensure the constant pool/jump table info is at least 4-byte aligned. 784193323Sed emitAlignment(16); 785193323Sed 786193323Sed emitConstantPool(F.getConstantPool()); 787203954Srdivacky if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) 788203954Srdivacky initJumpTableInfo(MJTI); 789193323Sed 790193323Sed // About to start emitting the machine code for the function. 791193323Sed emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); 792193323Sed TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); 793198892Srdivacky EmittedFunctions[F.getFunction()].Code = CurBufferPtr; 794193323Sed 795193323Sed MBBLocations.clear(); 796198090Srdivacky 797198090Srdivacky EmissionDetails.MF = &F; 798198090Srdivacky EmissionDetails.LineStarts.clear(); 799193323Sed} 800193323Sed 801193323Sedbool JITEmitter::finishFunction(MachineFunction &F) { 802193323Sed if (CurBufferPtr == BufferEnd) { 803198090Srdivacky // We must call endFunctionBody before retrying, because 804198090Srdivacky // deallocateMemForFunction requires it. 805198090Srdivacky MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 806198090Srdivacky retryWithMoreMemory(F); 807198090Srdivacky return true; 808193323Sed } 809198090Srdivacky 810203954Srdivacky if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) 811203954Srdivacky emitJumpTableInfo(MJTI); 812198090Srdivacky 813193323Sed // FnStart is the start of the text, not the start of the constant pool and 814193323Sed // other per-function data. 815193574Sed uint8_t *FnStart = 816193574Sed (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); 817193323Sed 818193323Sed // FnEnd is the end of the function's machine code. 819193574Sed uint8_t *FnEnd = CurBufferPtr; 820193323Sed 821193323Sed if (!Relocations.empty()) { 822193323Sed CurFn = F.getFunction(); 823193323Sed NumRelos += Relocations.size(); 824193323Sed 825193323Sed // Resolve the relocations to concrete pointers. 826193323Sed for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { 827193323Sed MachineRelocation &MR = Relocations[i]; 828193323Sed void *ResultPtr = 0; 829193323Sed if (!MR.letTargetResolve()) { 830193323Sed if (MR.isExternalSymbol()) { 831193323Sed ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), 832193323Sed false); 833202375Srdivacky DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" 834199481Srdivacky << ResultPtr << "]\n"); 835193323Sed 836193323Sed // If the target REALLY wants a stub for this function, emit it now. 837199481Srdivacky if (MR.mayNeedFarStub()) { 838199481Srdivacky ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); 839193323Sed } 840193323Sed } else if (MR.isGlobalValue()) { 841193323Sed ResultPtr = getPointerToGlobal(MR.getGlobalValue(), 842193323Sed BufferBegin+MR.getMachineCodeOffset(), 843199481Srdivacky MR.mayNeedFarStub()); 844193323Sed } else if (MR.isIndirectSymbol()) { 845199481Srdivacky ResultPtr = getPointerToGVIndirectSym( 846199481Srdivacky MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset()); 847193323Sed } else if (MR.isBasicBlock()) { 848193323Sed ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); 849193323Sed } else if (MR.isConstantPoolIndex()) { 850221345Sdim ResultPtr = 851221345Sdim (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); 852193323Sed } else { 853193323Sed assert(MR.isJumpTableIndex()); 854193323Sed ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); 855193323Sed } 856193323Sed 857193323Sed MR.setResultPointer(ResultPtr); 858193323Sed } 859193323Sed 860193323Sed // if we are managing the GOT and the relocation wants an index, 861193323Sed // give it one 862193323Sed if (MR.isGOTRelative() && MemMgr->isManagingGOT()) { 863193323Sed unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); 864193323Sed MR.setGOTIndex(idx); 865193323Sed if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { 866202375Srdivacky DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr 867198090Srdivacky << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 868198090Srdivacky << "\n"); 869193323Sed ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; 870193323Sed } 871193323Sed } 872193323Sed } 873193323Sed 874193323Sed CurFn = 0; 875193323Sed TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], 876193323Sed Relocations.size(), MemMgr->getGOTBase()); 877193323Sed } 878193323Sed 879193323Sed // Update the GOT entry for F to point to the new code. 880193323Sed if (MemMgr->isManagingGOT()) { 881193323Sed unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); 882193323Sed if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { 883202375Srdivacky DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin 884198090Srdivacky << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 885198090Srdivacky << "\n"); 886193323Sed ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; 887193323Sed } 888193323Sed } 889193323Sed 890193323Sed // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for 891193323Sed // global variables that were referenced in the relocations. 892193323Sed MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 893193323Sed 894193323Sed if (CurBufferPtr == BufferEnd) { 895198090Srdivacky retryWithMoreMemory(F); 896198090Srdivacky return true; 897198090Srdivacky } else { 898198090Srdivacky // Now that we've succeeded in emitting the function, reset the 899198090Srdivacky // SizeEstimate back down to zero. 900198090Srdivacky SizeEstimate = 0; 901193323Sed } 902193323Sed 903193323Sed BufferBegin = CurBufferPtr = 0; 904193323Sed NumBytes += FnEnd-FnStart; 905193323Sed 906193323Sed // Invalidate the icache if necessary. 907193323Sed sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart); 908193323Sed 909195098Sed TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, 910198090Srdivacky EmissionDetails); 911195098Sed 912207618Srdivacky // Reset the previous debug location. 913207618Srdivacky PrevDL = DebugLoc(); 914207618Srdivacky 915202375Srdivacky DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart 916245431Sdim << "] Function: " << F.getName() 917198090Srdivacky << ": " << (FnEnd-FnStart) << " bytes of text, " 918198090Srdivacky << Relocations.size() << " relocations\n"); 919193323Sed 920193323Sed Relocations.clear(); 921193323Sed ConstPoolAddresses.clear(); 922193323Sed 923193323Sed // Mark code region readable and executable if it's not so already. 924193323Sed MemMgr->setMemoryExecutable(); 925193323Sed 926207618Srdivacky DEBUG({ 927207618Srdivacky if (sys::hasDisassembler()) { 928207618Srdivacky dbgs() << "JIT: Disassembled code:\n"; 929207618Srdivacky dbgs() << sys::disassembleBuffer(FnStart, FnEnd-FnStart, 930207618Srdivacky (uintptr_t)FnStart); 931207618Srdivacky } else { 932207618Srdivacky dbgs() << "JIT: Binary code:\n"; 933207618Srdivacky uint8_t* q = FnStart; 934207618Srdivacky for (int i = 0; q < FnEnd; q += 4, ++i) { 935207618Srdivacky if (i == 4) 936207618Srdivacky i = 0; 937207618Srdivacky if (i == 0) 938207618Srdivacky dbgs() << "JIT: " << (long)(q - FnStart) << ": "; 939207618Srdivacky bool Done = false; 940207618Srdivacky for (int j = 3; j >= 0; --j) { 941207618Srdivacky if (q + j >= FnEnd) 942207618Srdivacky Done = true; 943207618Srdivacky else 944207618Srdivacky dbgs() << (unsigned short)q[j]; 945207618Srdivacky } 946207618Srdivacky if (Done) 947207618Srdivacky break; 948207618Srdivacky dbgs() << ' '; 949207618Srdivacky if (i == 3) 950207618Srdivacky dbgs() << '\n'; 951193323Sed } 952207618Srdivacky dbgs()<< '\n'; 953193323Sed } 954207618Srdivacky }); 955198090Srdivacky 956193323Sed if (MMI) 957193323Sed MMI->EndFunction(); 958199481Srdivacky 959193323Sed return false; 960193323Sed} 961193323Sed 962198090Srdivackyvoid JITEmitter::retryWithMoreMemory(MachineFunction &F) { 963202375Srdivacky DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n"); 964198090Srdivacky Relocations.clear(); // Clear the old relocations or we'll reapply them. 965198090Srdivacky ConstPoolAddresses.clear(); 966198090Srdivacky ++NumRetries; 967198090Srdivacky deallocateMemForFunction(F.getFunction()); 968198090Srdivacky // Try again with at least twice as much free space. 969198090Srdivacky SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); 970210299Sed 971210299Sed for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){ 972210299Sed if (MBB->hasAddressTaken()) 973210299Sed TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock()); 974210299Sed } 975198090Srdivacky} 976198090Srdivacky 977193323Sed/// deallocateMemForFunction - Deallocate all memory for the specified 978193323Sed/// function body. Also drop any references the function has to stubs. 979198892Srdivacky/// May be called while the Function is being destroyed inside ~Value(). 980198090Srdivackyvoid JITEmitter::deallocateMemForFunction(const Function *F) { 981198892Srdivacky ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator 982198892Srdivacky Emitted = EmittedFunctions.find(F); 983198396Srdivacky if (Emitted != EmittedFunctions.end()) { 984198396Srdivacky MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody); 985198892Srdivacky TheJIT->NotifyFreeingMachineCode(Emitted->second.Code); 986198892Srdivacky 987198396Srdivacky EmittedFunctions.erase(Emitted); 988198396Srdivacky } 989193323Sed} 990193323Sed 991193323Sed 992235633Sdimvoid *JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { 993193323Sed if (BufferBegin) 994193323Sed return JITCodeEmitter::allocateSpace(Size, Alignment); 995193323Sed 996193323Sed // create a new memory block if there is no active one. 997193323Sed // care must be taken so that BufferBegin is invalidated when a 998193323Sed // block is trimmed 999193323Sed BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment); 1000193323Sed BufferEnd = BufferBegin+Size; 1001193323Sed return CurBufferPtr; 1002193323Sed} 1003193323Sed 1004235633Sdimvoid *JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { 1005198090Srdivacky // Delegate this call through the memory manager. 1006198090Srdivacky return MemMgr->allocateGlobal(Size, Alignment); 1007198090Srdivacky} 1008198090Srdivacky 1009193323Sedvoid JITEmitter::emitConstantPool(MachineConstantPool *MCP) { 1010193323Sed if (TheJIT->getJITInfo().hasCustomConstantPool()) 1011193323Sed return; 1012193323Sed 1013193323Sed const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 1014193323Sed if (Constants.empty()) return; 1015193323Sed 1016245431Sdim unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getDataLayout()); 1017193323Sed unsigned Align = MCP->getConstantPoolAlignment(); 1018193323Sed ConstantPoolBase = allocateSpace(Size, Align); 1019193323Sed ConstantPool = MCP; 1020193323Sed 1021193323Sed if (ConstantPoolBase == 0) return; // Buffer overflow. 1022193323Sed 1023202375Srdivacky DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase 1024198090Srdivacky << "] (size: " << Size << ", alignment: " << Align << ")\n"); 1025193323Sed 1026193323Sed // Initialize the memory for all of the constant pool entries. 1027193323Sed unsigned Offset = 0; 1028193323Sed for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 1029193323Sed MachineConstantPoolEntry CPE = Constants[i]; 1030193323Sed unsigned AlignMask = CPE.getAlignment() - 1; 1031193323Sed Offset = (Offset + AlignMask) & ~AlignMask; 1032193323Sed 1033193323Sed uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset; 1034193323Sed ConstPoolAddresses.push_back(CAddr); 1035193323Sed if (CPE.isMachineConstantPoolEntry()) { 1036193323Sed // FIXME: add support to lower machine constant pool values into bytes! 1037207618Srdivacky report_fatal_error("Initialize memory with machine specific constant pool" 1038198090Srdivacky "entry has not been implemented!"); 1039193323Sed } 1040193323Sed TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); 1041202375Srdivacky DEBUG(dbgs() << "JIT: CP" << i << " at [0x"; 1042202375Srdivacky dbgs().write_hex(CAddr) << "]\n"); 1043193323Sed 1044226890Sdim Type *Ty = CPE.Val.ConstVal->getType(); 1045245431Sdim Offset += TheJIT->getDataLayout()->getTypeAllocSize(Ty); 1046193323Sed } 1047193323Sed} 1048193323Sed 1049193323Sedvoid JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { 1050193323Sed if (TheJIT->getJITInfo().hasCustomJumpTables()) 1051193323Sed return; 1052205218Srdivacky if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) 1053205218Srdivacky return; 1054193323Sed 1055193323Sed const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1056193323Sed if (JT.empty()) return; 1057199481Srdivacky 1058193323Sed unsigned NumEntries = 0; 1059193323Sed for (unsigned i = 0, e = JT.size(); i != e; ++i) 1060193323Sed NumEntries += JT[i].MBBs.size(); 1061193323Sed 1062245431Sdim unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getDataLayout()); 1063193323Sed 1064193323Sed // Just allocate space for all the jump tables now. We will fix up the actual 1065193323Sed // MBB entries in the tables after we emit the code for each block, since then 1066193323Sed // we will know the final locations of the MBBs in memory. 1067193323Sed JumpTable = MJTI; 1068203954Srdivacky JumpTableBase = allocateSpace(NumEntries * EntrySize, 1069245431Sdim MJTI->getEntryAlignment(*TheJIT->getDataLayout())); 1070193323Sed} 1071193323Sed 1072193323Sedvoid JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { 1073193323Sed if (TheJIT->getJITInfo().hasCustomJumpTables()) 1074193323Sed return; 1075193323Sed 1076193323Sed const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1077193323Sed if (JT.empty() || JumpTableBase == 0) return; 1078199481Srdivacky 1079221345Sdim 1080203954Srdivacky switch (MJTI->getEntryKind()) { 1081205218Srdivacky case MachineJumpTableInfo::EK_Inline: 1082205218Srdivacky return; 1083203954Srdivacky case MachineJumpTableInfo::EK_BlockAddress: { 1084203954Srdivacky // EK_BlockAddress - Each entry is a plain address of block, e.g.: 1085203954Srdivacky // .word LBB123 1086245431Sdim assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == sizeof(void*) && 1087203954Srdivacky "Cross JIT'ing?"); 1088221345Sdim 1089203954Srdivacky // For each jump table, map each target in the jump table to the address of 1090203954Srdivacky // an emitted MachineBasicBlock. 1091203954Srdivacky intptr_t *SlotPtr = (intptr_t*)JumpTableBase; 1092221345Sdim 1093203954Srdivacky for (unsigned i = 0, e = JT.size(); i != e; ++i) { 1094203954Srdivacky const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 1095203954Srdivacky // Store the address of the basic block for this jump table slot in the 1096203954Srdivacky // memory we allocated for the jump table in 'initJumpTableInfo' 1097203954Srdivacky for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) 1098203954Srdivacky *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); 1099203954Srdivacky } 1100203954Srdivacky break; 1101203954Srdivacky } 1102221345Sdim 1103203954Srdivacky case MachineJumpTableInfo::EK_Custom32: 1104203954Srdivacky case MachineJumpTableInfo::EK_GPRel32BlockAddress: 1105203954Srdivacky case MachineJumpTableInfo::EK_LabelDifference32: { 1106245431Sdim assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == 4&&"Cross JIT'ing?"); 1107193323Sed // For each jump table, place the offset from the beginning of the table 1108193323Sed // to the target address. 1109193323Sed int *SlotPtr = (int*)JumpTableBase; 1110193323Sed 1111193323Sed for (unsigned i = 0, e = JT.size(); i != e; ++i) { 1112193323Sed const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 1113193323Sed // Store the offset of the basic block for this jump table slot in the 1114193323Sed // memory we allocated for the jump table in 'initJumpTableInfo' 1115193323Sed uintptr_t Base = (uintptr_t)SlotPtr; 1116193323Sed for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { 1117193323Sed uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); 1118203954Srdivacky /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook. 1119193323Sed *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); 1120193323Sed } 1121193323Sed } 1122203954Srdivacky break; 1123193323Sed } 1124235633Sdim case MachineJumpTableInfo::EK_GPRel64BlockAddress: 1125235633Sdim llvm_unreachable( 1126235633Sdim "JT Info emission not implemented for GPRel64BlockAddress yet."); 1127203954Srdivacky } 1128193323Sed} 1129193323Sed 1130201360Srdivackyvoid JITEmitter::startGVStub(const GlobalValue* GV, 1131199989Srdivacky unsigned StubSize, unsigned Alignment) { 1132201360Srdivacky SavedBufferBegin = BufferBegin; 1133201360Srdivacky SavedBufferEnd = BufferEnd; 1134201360Srdivacky SavedCurBufferPtr = CurBufferPtr; 1135199481Srdivacky 1136193323Sed BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); 1137193323Sed BufferEnd = BufferBegin+StubSize+1; 1138193323Sed} 1139193323Sed 1140201360Srdivackyvoid JITEmitter::startGVStub(void *Buffer, unsigned StubSize) { 1141201360Srdivacky SavedBufferBegin = BufferBegin; 1142201360Srdivacky SavedBufferEnd = BufferEnd; 1143201360Srdivacky SavedCurBufferPtr = CurBufferPtr; 1144199481Srdivacky 1145193574Sed BufferBegin = CurBufferPtr = (uint8_t *)Buffer; 1146193323Sed BufferEnd = BufferBegin+StubSize+1; 1147193323Sed} 1148193323Sed 1149201360Srdivackyvoid JITEmitter::finishGVStub() { 1150199989Srdivacky assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space."); 1151193323Sed NumBytes += getCurrentPCOffset(); 1152201360Srdivacky BufferBegin = SavedBufferBegin; 1153201360Srdivacky BufferEnd = SavedBufferEnd; 1154201360Srdivacky CurBufferPtr = SavedCurBufferPtr; 1155193323Sed} 1156193323Sed 1157201360Srdivackyvoid *JITEmitter::allocIndirectGV(const GlobalValue *GV, 1158201360Srdivacky const uint8_t *Buffer, size_t Size, 1159201360Srdivacky unsigned Alignment) { 1160201360Srdivacky uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment); 1161201360Srdivacky memcpy(IndGV, Buffer, Size); 1162201360Srdivacky return IndGV; 1163201360Srdivacky} 1164201360Srdivacky 1165193323Sed// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry 1166193323Sed// in the constant pool that was last emitted with the 'emitConstantPool' 1167193323Sed// method. 1168193323Sed// 1169193323Seduintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { 1170193323Sed assert(ConstantNum < ConstantPool->getConstants().size() && 1171193323Sed "Invalid ConstantPoolIndex!"); 1172193323Sed return ConstPoolAddresses[ConstantNum]; 1173193323Sed} 1174193323Sed 1175193323Sed// getJumpTableEntryAddress - Return the address of the JumpTable with index 1176193323Sed// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' 1177193323Sed// 1178193323Seduintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { 1179193323Sed const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); 1180193323Sed assert(Index < JT.size() && "Invalid jump table index!"); 1181199481Srdivacky 1182245431Sdim unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getDataLayout()); 1183203954Srdivacky 1184193323Sed unsigned Offset = 0; 1185193323Sed for (unsigned i = 0; i < Index; ++i) 1186193323Sed Offset += JT[i].MBBs.size(); 1187199481Srdivacky 1188193323Sed Offset *= EntrySize; 1189199481Srdivacky 1190193323Sed return (uintptr_t)((char *)JumpTableBase + Offset); 1191193323Sed} 1192193323Sed 1193198892Srdivackyvoid JITEmitter::EmittedFunctionConfig::onDelete( 1194198892Srdivacky JITEmitter *Emitter, const Function *F) { 1195198892Srdivacky Emitter->deallocateMemForFunction(F); 1196198892Srdivacky} 1197198892Srdivackyvoid JITEmitter::EmittedFunctionConfig::onRAUW( 1198198892Srdivacky JITEmitter *, const Function*, const Function*) { 1199198892Srdivacky llvm_unreachable("The JIT doesn't know how to handle a" 1200198892Srdivacky " RAUW on a value it has emitted."); 1201198892Srdivacky} 1202198892Srdivacky 1203198892Srdivacky 1204193323Sed//===----------------------------------------------------------------------===// 1205193323Sed// Public interface to this file 1206193323Sed//===----------------------------------------------------------------------===// 1207193323Sed 1208198090SrdivackyJITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM, 1209198090Srdivacky TargetMachine &tm) { 1210198090Srdivacky return new JITEmitter(jit, JMM, tm); 1211193323Sed} 1212193323Sed 1213193323Sed// getPointerToFunctionOrStub - If the specified function has been 1214193323Sed// code-gen'd, return a pointer to the function. If not, compile it, or use 1215193323Sed// a stub to implement lazy compilation if available. 1216193323Sed// 1217193323Sedvoid *JIT::getPointerToFunctionOrStub(Function *F) { 1218193323Sed // If we have already code generated the function, just return the address. 1219193323Sed if (void *Addr = getPointerToGlobalIfAvailable(F)) 1220193323Sed return Addr; 1221199481Srdivacky 1222193323Sed // Get a stub if the target supports it. 1223245431Sdim JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter()); 1224199989Srdivacky return JE->getJITResolver().getLazyFunctionStub(F); 1225193323Sed} 1226193323Sed 1227193323Sedvoid JIT::updateFunctionStub(Function *F) { 1228193323Sed // Get the empty stub we generated earlier. 1229245431Sdim JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter()); 1230199989Srdivacky void *Stub = JE->getJITResolver().getLazyFunctionStub(F); 1231199989Srdivacky void *Addr = getPointerToGlobalIfAvailable(F); 1232201360Srdivacky assert(Addr != Stub && "Function must have non-stub address to be updated."); 1233193323Sed 1234193323Sed // Tell the target jit info to rewrite the stub at the specified address, 1235193323Sed // rather than creating a new one. 1236199989Srdivacky TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout(); 1237201360Srdivacky JE->startGVStub(Stub, layout.Size); 1238199989Srdivacky getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter()); 1239201360Srdivacky JE->finishGVStub(); 1240193323Sed} 1241193323Sed 1242193323Sed/// freeMachineCodeForFunction - release machine code memory for given Function. 1243193323Sed/// 1244193323Sedvoid JIT::freeMachineCodeForFunction(Function *F) { 1245193323Sed // Delete translation for this from the ExecutionEngine, so it will get 1246193323Sed // retranslated next time it is used. 1247198892Srdivacky updateGlobalMapping(F, 0); 1248193323Sed 1249193323Sed // Free the actual memory for the function body and related stuff. 1250245431Sdim static_cast<JITEmitter*>(JCE)->deallocateMemForFunction(F); 1251193323Sed} 1252