JITEmitter.cpp revision 210299
1193323Sed//===-- JITEmitter.cpp - Write machine code to executable memory ----------===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This file defines a MachineCodeEmitter object that is used by the JIT to 11193323Sed// write machine code to memory and remember where relocatable values are. 12193323Sed// 13193323Sed//===----------------------------------------------------------------------===// 14193323Sed 15193323Sed#define DEBUG_TYPE "jit" 16193323Sed#include "JIT.h" 17198090Srdivacky#include "JITDebugRegisterer.h" 18193323Sed#include "JITDwarfEmitter.h" 19198090Srdivacky#include "llvm/ADT/OwningPtr.h" 20193323Sed#include "llvm/Constants.h" 21193323Sed#include "llvm/Module.h" 22193323Sed#include "llvm/DerivedTypes.h" 23202878Srdivacky#include "llvm/Analysis/DebugInfo.h" 24193323Sed#include "llvm/CodeGen/JITCodeEmitter.h" 25193323Sed#include "llvm/CodeGen/MachineFunction.h" 26207618Srdivacky#include "llvm/CodeGen/MachineCodeInfo.h" 27193323Sed#include "llvm/CodeGen/MachineConstantPool.h" 28193323Sed#include "llvm/CodeGen/MachineJumpTableInfo.h" 29193323Sed#include "llvm/CodeGen/MachineModuleInfo.h" 30193323Sed#include "llvm/CodeGen/MachineRelocation.h" 31195098Sed#include "llvm/ExecutionEngine/GenericValue.h" 32195098Sed#include "llvm/ExecutionEngine/JITEventListener.h" 33193323Sed#include "llvm/ExecutionEngine/JITMemoryManager.h" 34193323Sed#include "llvm/Target/TargetData.h" 35207618Srdivacky#include "llvm/Target/TargetInstrInfo.h" 36193323Sed#include "llvm/Target/TargetJITInfo.h" 37193323Sed#include "llvm/Target/TargetMachine.h" 38193323Sed#include "llvm/Target/TargetOptions.h" 39193323Sed#include "llvm/Support/Debug.h" 40198090Srdivacky#include "llvm/Support/ErrorHandling.h" 41203954Srdivacky#include "llvm/Support/ManagedStatic.h" 42193323Sed#include "llvm/Support/MutexGuard.h" 43193323Sed#include "llvm/Support/ValueHandle.h" 44198090Srdivacky#include "llvm/Support/raw_ostream.h" 45193323Sed#include "llvm/System/Disassembler.h" 46193323Sed#include "llvm/System/Memory.h" 47198396Srdivacky#include "llvm/ADT/DenseMap.h" 48193323Sed#include "llvm/ADT/SmallPtrSet.h" 49193323Sed#include "llvm/ADT/SmallVector.h" 50193323Sed#include "llvm/ADT/Statistic.h" 51198892Srdivacky#include "llvm/ADT/ValueMap.h" 52193323Sed#include <algorithm> 53193323Sed#ifndef NDEBUG 54193323Sed#include <iomanip> 55193323Sed#endif 56193323Sedusing namespace llvm; 57193323Sed 58193323SedSTATISTIC(NumBytes, "Number of bytes of machine code compiled"); 59193323SedSTATISTIC(NumRelos, "Number of relocations applied"); 60198090SrdivackySTATISTIC(NumRetries, "Number of retries with more memory"); 61193323Sed 62193323Sed 63201360Srdivacky// A declaration may stop being a declaration once it's fully read from bitcode. 64201360Srdivacky// This function returns true if F is fully read and is still a declaration. 65201360Srdivackystatic bool isNonGhostDeclaration(const Function *F) { 66203954Srdivacky return F->isDeclaration() && !F->isMaterializable(); 67201360Srdivacky} 68201360Srdivacky 69193323Sed//===----------------------------------------------------------------------===// 70193323Sed// JIT lazy compilation code. 71193323Sed// 72193323Sednamespace { 73199481Srdivacky class JITEmitter; 74198892Srdivacky class JITResolverState; 75198892Srdivacky 76198892Srdivacky template<typename ValueTy> 77198892Srdivacky struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> { 78198892Srdivacky typedef JITResolverState *ExtraData; 79198892Srdivacky static void onRAUW(JITResolverState *, Value *Old, Value *New) { 80198892Srdivacky assert(false && "The JIT doesn't know how to handle a" 81198892Srdivacky " RAUW on a value it has emitted."); 82198892Srdivacky } 83198892Srdivacky }; 84198892Srdivacky 85198892Srdivacky struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> { 86198892Srdivacky typedef JITResolverState *ExtraData; 87198892Srdivacky static void onDelete(JITResolverState *JRS, Function *F); 88198892Srdivacky }; 89198892Srdivacky 90193323Sed class JITResolverState { 91193323Sed public: 92198892Srdivacky typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> > 93199989Srdivacky FunctionToLazyStubMapTy; 94198396Srdivacky typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy; 95198892Srdivacky typedef ValueMap<Function *, SmallPtrSet<void*, 1>, 96198892Srdivacky CallSiteValueMapConfig> FunctionToCallSitesMapTy; 97193323Sed typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy; 98193323Sed private: 99199989Srdivacky /// FunctionToLazyStubMap - Keep track of the lazy stub created for a 100199989Srdivacky /// particular function so that we can reuse them if necessary. 101199989Srdivacky FunctionToLazyStubMapTy FunctionToLazyStubMap; 102193323Sed 103198396Srdivacky /// CallSiteToFunctionMap - Keep track of the function that each lazy call 104198396Srdivacky /// site corresponds to, and vice versa. 105198396Srdivacky CallSiteToFunctionMapTy CallSiteToFunctionMap; 106198396Srdivacky FunctionToCallSitesMapTy FunctionToCallSitesMap; 107193323Sed 108193323Sed /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a 109193323Sed /// particular GlobalVariable so that we can reuse them if necessary. 110193323Sed GlobalToIndirectSymMapTy GlobalToIndirectSymMap; 111193323Sed 112203954Srdivacky /// Instance of the JIT this ResolverState serves. 113203954Srdivacky JIT *TheJIT; 114203954Srdivacky 115193323Sed public: 116203954Srdivacky JITResolverState(JIT *jit) : FunctionToLazyStubMap(this), 117203954Srdivacky FunctionToCallSitesMap(this), 118203954Srdivacky TheJIT(jit) {} 119198892Srdivacky 120199989Srdivacky FunctionToLazyStubMapTy& getFunctionToLazyStubMap( 121199989Srdivacky const MutexGuard& locked) { 122193323Sed assert(locked.holds(TheJIT->lock)); 123199989Srdivacky return FunctionToLazyStubMap; 124193323Sed } 125193323Sed 126198396Srdivacky GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap(const MutexGuard& locked) { 127193323Sed assert(locked.holds(TheJIT->lock)); 128198396Srdivacky return GlobalToIndirectSymMap; 129193323Sed } 130193323Sed 131198396Srdivacky pair<void *, Function *> LookupFunctionFromCallSite( 132198396Srdivacky const MutexGuard &locked, void *CallSite) const { 133193323Sed assert(locked.holds(TheJIT->lock)); 134198396Srdivacky 135198396Srdivacky // The address given to us for the stub may not be exactly right, it might be 136198396Srdivacky // a little bit after the stub. As such, use upper_bound to find it. 137198396Srdivacky CallSiteToFunctionMapTy::const_iterator I = 138198396Srdivacky CallSiteToFunctionMap.upper_bound(CallSite); 139198396Srdivacky assert(I != CallSiteToFunctionMap.begin() && 140198396Srdivacky "This is not a known call site!"); 141198396Srdivacky --I; 142198396Srdivacky return *I; 143193323Sed } 144198396Srdivacky 145198396Srdivacky void AddCallSite(const MutexGuard &locked, void *CallSite, Function *F) { 146198396Srdivacky assert(locked.holds(TheJIT->lock)); 147198396Srdivacky 148198892Srdivacky bool Inserted = CallSiteToFunctionMap.insert( 149198892Srdivacky std::make_pair(CallSite, F)).second; 150198892Srdivacky (void)Inserted; 151198892Srdivacky assert(Inserted && "Pair was already in CallSiteToFunctionMap"); 152198396Srdivacky FunctionToCallSitesMap[F].insert(CallSite); 153198396Srdivacky } 154198396Srdivacky 155198396Srdivacky // Returns the Function of the stub if a stub was erased, or NULL if there 156198396Srdivacky // was no stub. This function uses the call-site->function map to find a 157198396Srdivacky // relevant function, but asserts that only stubs and not other call sites 158198396Srdivacky // will be passed in. 159204792Srdivacky Function *EraseStub(const MutexGuard &locked, void *Stub); 160198396Srdivacky 161204792Srdivacky void EraseAllCallSitesFor(const MutexGuard &locked, Function *F) { 162204792Srdivacky assert(locked.holds(TheJIT->lock)); 163204792Srdivacky EraseAllCallSitesForPrelocked(F); 164198396Srdivacky } 165204792Srdivacky void EraseAllCallSitesForPrelocked(Function *F); 166198396Srdivacky 167204792Srdivacky // Erases _all_ call sites regardless of their function. This is used to 168204792Srdivacky // unregister the stub addresses from the StubToResolverMap in 169204792Srdivacky // ~JITResolver(). 170204792Srdivacky void EraseAllCallSitesPrelocked(); 171193323Sed }; 172193323Sed 173193323Sed /// JITResolver - Keep track of, and resolve, call sites for functions that 174193323Sed /// have not yet been compiled. 175193323Sed class JITResolver { 176199989Srdivacky typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy; 177198396Srdivacky typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy; 178193323Sed typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; 179193323Sed 180193323Sed /// LazyResolverFn - The target lazy resolver function that we actually 181193323Sed /// rewrite instructions to use. 182193323Sed TargetJITInfo::LazyResolverFn LazyResolverFn; 183193323Sed 184193323Sed JITResolverState state; 185193323Sed 186199989Srdivacky /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap 187199989Srdivacky /// for external functions. TODO: Of course, external functions don't need 188199989Srdivacky /// a lazy stub. It's actually here to make it more likely that far calls 189199989Srdivacky /// succeed, but no single stub can guarantee that. I'll remove this in a 190199989Srdivacky /// subsequent checkin when I actually fix far calls. 191193323Sed std::map<void*, void*> ExternalFnToStubMap; 192193323Sed 193193323Sed /// revGOTMap - map addresses to indexes in the GOT 194193323Sed std::map<void*, unsigned> revGOTMap; 195193323Sed unsigned nextGOTIndex; 196193323Sed 197199481Srdivacky JITEmitter &JE; 198199481Srdivacky 199203954Srdivacky /// Instance of JIT corresponding to this Resolver. 200203954Srdivacky JIT *TheJIT; 201203954Srdivacky 202193323Sed public: 203203954Srdivacky explicit JITResolver(JIT &jit, JITEmitter &je) 204203954Srdivacky : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) { 205193323Sed LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); 206193323Sed } 207199481Srdivacky 208204792Srdivacky ~JITResolver(); 209204792Srdivacky 210199989Srdivacky /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's 211199989Srdivacky /// lazy-compilation stub if it has already been created. 212199989Srdivacky void *getLazyFunctionStubIfAvailable(Function *F); 213193323Sed 214199989Srdivacky /// getLazyFunctionStub - This returns a pointer to a function's 215199989Srdivacky /// lazy-compilation stub, creating one on demand as needed. 216199989Srdivacky void *getLazyFunctionStub(Function *F); 217193323Sed 218193323Sed /// getExternalFunctionStub - Return a stub for the function at the 219193323Sed /// specified address, created lazily on demand. 220193323Sed void *getExternalFunctionStub(void *FnAddr); 221193323Sed 222193323Sed /// getGlobalValueIndirectSym - Return an indirect symbol containing the 223193323Sed /// specified GV address. 224193323Sed void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); 225193323Sed 226193323Sed void getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs, 227193323Sed SmallVectorImpl<void*> &Ptrs); 228199481Srdivacky 229193323Sed /// getGOTIndexForAddress - Return a new or existing index in the GOT for 230193323Sed /// an address. This function only manages slots, it does not manage the 231193323Sed /// contents of the slots or the memory associated with the GOT. 232193323Sed unsigned getGOTIndexForAddr(void *addr); 233193323Sed 234193323Sed /// JITCompilerFn - This function is called to resolve a stub to a compiled 235193323Sed /// address. If the LLVM Function corresponding to the stub has not yet 236193323Sed /// been compiled, this function compiles it first. 237193323Sed static void *JITCompilerFn(void *Stub); 238193323Sed }; 239199481Srdivacky 240203954Srdivacky class StubToResolverMapTy { 241203954Srdivacky /// Map a stub address to a specific instance of a JITResolver so that 242203954Srdivacky /// lazily-compiled functions can find the right resolver to use. 243203954Srdivacky /// 244203954Srdivacky /// Guarded by Lock. 245203954Srdivacky std::map<void*, JITResolver*> Map; 246203954Srdivacky 247203954Srdivacky /// Guards Map from concurrent accesses. 248203954Srdivacky mutable sys::Mutex Lock; 249203954Srdivacky 250203954Srdivacky public: 251203954Srdivacky /// Registers a Stub to be resolved by Resolver. 252203954Srdivacky void RegisterStubResolver(void *Stub, JITResolver *Resolver) { 253203954Srdivacky MutexGuard guard(Lock); 254203954Srdivacky Map.insert(std::make_pair(Stub, Resolver)); 255203954Srdivacky } 256203954Srdivacky /// Unregisters the Stub when it's invalidated. 257203954Srdivacky void UnregisterStubResolver(void *Stub) { 258203954Srdivacky MutexGuard guard(Lock); 259203954Srdivacky Map.erase(Stub); 260203954Srdivacky } 261203954Srdivacky /// Returns the JITResolver instance that owns the Stub. 262203954Srdivacky JITResolver *getResolverFromStub(void *Stub) const { 263203954Srdivacky MutexGuard guard(Lock); 264203954Srdivacky // The address given to us for the stub may not be exactly right, it might 265203954Srdivacky // be a little bit after the stub. As such, use upper_bound to find it. 266203954Srdivacky // This is the same trick as in LookupFunctionFromCallSite from 267203954Srdivacky // JITResolverState. 268203954Srdivacky std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub); 269203954Srdivacky assert(I != Map.begin() && "This is not a known stub!"); 270203954Srdivacky --I; 271203954Srdivacky return I->second; 272203954Srdivacky } 273204792Srdivacky /// True if any stubs refer to the given resolver. Only used in an assert(). 274204792Srdivacky /// O(N) 275204792Srdivacky bool ResolverHasStubs(JITResolver* Resolver) const { 276204792Srdivacky MutexGuard guard(Lock); 277204792Srdivacky for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(), 278204792Srdivacky E = Map.end(); I != E; ++I) { 279204792Srdivacky if (I->second == Resolver) 280204792Srdivacky return true; 281204792Srdivacky } 282204792Srdivacky return false; 283204792Srdivacky } 284203954Srdivacky }; 285203954Srdivacky /// This needs to be static so that a lazy call stub can access it with no 286203954Srdivacky /// context except the address of the stub. 287203954Srdivacky ManagedStatic<StubToResolverMapTy> StubToResolverMap; 288203954Srdivacky 289199481Srdivacky /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is 290199481Srdivacky /// used to output functions to memory for execution. 291199481Srdivacky class JITEmitter : public JITCodeEmitter { 292199481Srdivacky JITMemoryManager *MemMgr; 293199481Srdivacky 294201360Srdivacky // When outputting a function stub in the context of some other function, we 295201360Srdivacky // save BufferBegin/BufferEnd/CurBufferPtr here. 296201360Srdivacky uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; 297201360Srdivacky 298199481Srdivacky // When reattempting to JIT a function after running out of space, we store 299199481Srdivacky // the estimated size of the function we're trying to JIT here, so we can 300199481Srdivacky // ask the memory manager for at least this much space. When we 301199481Srdivacky // successfully emit the function, we reset this back to zero. 302199481Srdivacky uintptr_t SizeEstimate; 303199481Srdivacky 304199481Srdivacky /// Relocations - These are the relocations that the function needs, as 305199481Srdivacky /// emitted. 306199481Srdivacky std::vector<MachineRelocation> Relocations; 307199481Srdivacky 308199481Srdivacky /// MBBLocations - This vector is a mapping from MBB ID's to their address. 309199481Srdivacky /// It is filled in by the StartMachineBasicBlock callback and queried by 310199481Srdivacky /// the getMachineBasicBlockAddress callback. 311199481Srdivacky std::vector<uintptr_t> MBBLocations; 312199481Srdivacky 313199481Srdivacky /// ConstantPool - The constant pool for the current function. 314199481Srdivacky /// 315199481Srdivacky MachineConstantPool *ConstantPool; 316199481Srdivacky 317199481Srdivacky /// ConstantPoolBase - A pointer to the first entry in the constant pool. 318199481Srdivacky /// 319199481Srdivacky void *ConstantPoolBase; 320199481Srdivacky 321199481Srdivacky /// ConstPoolAddresses - Addresses of individual constant pool entries. 322199481Srdivacky /// 323199481Srdivacky SmallVector<uintptr_t, 8> ConstPoolAddresses; 324199481Srdivacky 325199481Srdivacky /// JumpTable - The jump tables for the current function. 326199481Srdivacky /// 327199481Srdivacky MachineJumpTableInfo *JumpTable; 328199481Srdivacky 329199481Srdivacky /// JumpTableBase - A pointer to the first entry in the jump table. 330199481Srdivacky /// 331199481Srdivacky void *JumpTableBase; 332199481Srdivacky 333199481Srdivacky /// Resolver - This contains info about the currently resolved functions. 334199481Srdivacky JITResolver Resolver; 335199481Srdivacky 336199481Srdivacky /// DE - The dwarf emitter for the jit. 337199481Srdivacky OwningPtr<JITDwarfEmitter> DE; 338199481Srdivacky 339199481Srdivacky /// DR - The debug registerer for the jit. 340199481Srdivacky OwningPtr<JITDebugRegisterer> DR; 341199481Srdivacky 342199481Srdivacky /// LabelLocations - This vector is a mapping from Label ID's to their 343199481Srdivacky /// address. 344205218Srdivacky DenseMap<MCSymbol*, uintptr_t> LabelLocations; 345199481Srdivacky 346199481Srdivacky /// MMI - Machine module info for exception informations 347199481Srdivacky MachineModuleInfo* MMI; 348199481Srdivacky 349199481Srdivacky // CurFn - The llvm function being emitted. Only valid during 350199481Srdivacky // finishFunction(). 351199481Srdivacky const Function *CurFn; 352199481Srdivacky 353199481Srdivacky /// Information about emitted code, which is passed to the 354199481Srdivacky /// JITEventListeners. This is reset in startFunction and used in 355199481Srdivacky /// finishFunction. 356199481Srdivacky JITEvent_EmittedFunctionDetails EmissionDetails; 357199481Srdivacky 358199481Srdivacky struct EmittedCode { 359199481Srdivacky void *FunctionBody; // Beginning of the function's allocation. 360199481Srdivacky void *Code; // The address the function's code actually starts at. 361199481Srdivacky void *ExceptionTable; 362199481Srdivacky EmittedCode() : FunctionBody(0), Code(0), ExceptionTable(0) {} 363199481Srdivacky }; 364199481Srdivacky struct EmittedFunctionConfig : public ValueMapConfig<const Function*> { 365199481Srdivacky typedef JITEmitter *ExtraData; 366199481Srdivacky static void onDelete(JITEmitter *, const Function*); 367199481Srdivacky static void onRAUW(JITEmitter *, const Function*, const Function*); 368199481Srdivacky }; 369199481Srdivacky ValueMap<const Function *, EmittedCode, 370199481Srdivacky EmittedFunctionConfig> EmittedFunctions; 371199481Srdivacky 372207618Srdivacky DebugLoc PrevDL; 373199481Srdivacky 374203954Srdivacky /// Instance of the JIT 375203954Srdivacky JIT *TheJIT; 376203954Srdivacky 377199481Srdivacky public: 378199481Srdivacky JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM) 379199481Srdivacky : SizeEstimate(0), Resolver(jit, *this), MMI(0), CurFn(0), 380207618Srdivacky EmittedFunctions(this), TheJIT(&jit) { 381199481Srdivacky MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); 382199481Srdivacky if (jit.getJITInfo().needsGOT()) { 383199481Srdivacky MemMgr->AllocateGOT(); 384202375Srdivacky DEBUG(dbgs() << "JIT is managing a GOT\n"); 385199481Srdivacky } 386199481Srdivacky 387207618Srdivacky if (JITExceptionHandling || JITEmitDebugInfo) { 388199481Srdivacky DE.reset(new JITDwarfEmitter(jit)); 389199481Srdivacky } 390199481Srdivacky if (JITEmitDebugInfo) { 391199481Srdivacky DR.reset(new JITDebugRegisterer(TM)); 392199481Srdivacky } 393199481Srdivacky } 394199481Srdivacky ~JITEmitter() { 395199481Srdivacky delete MemMgr; 396199481Srdivacky } 397199481Srdivacky 398199481Srdivacky /// classof - Methods for support type inquiry through isa, cast, and 399199481Srdivacky /// dyn_cast: 400199481Srdivacky /// 401199481Srdivacky static inline bool classof(const JITEmitter*) { return true; } 402199481Srdivacky static inline bool classof(const MachineCodeEmitter*) { return true; } 403199481Srdivacky 404199481Srdivacky JITResolver &getJITResolver() { return Resolver; } 405199481Srdivacky 406199481Srdivacky virtual void startFunction(MachineFunction &F); 407199481Srdivacky virtual bool finishFunction(MachineFunction &F); 408199481Srdivacky 409199481Srdivacky void emitConstantPool(MachineConstantPool *MCP); 410199481Srdivacky void initJumpTableInfo(MachineJumpTableInfo *MJTI); 411199481Srdivacky void emitJumpTableInfo(MachineJumpTableInfo *MJTI); 412199481Srdivacky 413201360Srdivacky void startGVStub(const GlobalValue* GV, 414201360Srdivacky unsigned StubSize, unsigned Alignment = 1); 415201360Srdivacky void startGVStub(void *Buffer, unsigned StubSize); 416201360Srdivacky void finishGVStub(); 417201360Srdivacky virtual void *allocIndirectGV(const GlobalValue *GV, 418201360Srdivacky const uint8_t *Buffer, size_t Size, 419201360Srdivacky unsigned Alignment); 420199481Srdivacky 421199481Srdivacky /// allocateSpace - Reserves space in the current block if any, or 422199481Srdivacky /// allocate a new one of the given size. 423199481Srdivacky virtual void *allocateSpace(uintptr_t Size, unsigned Alignment); 424199481Srdivacky 425199481Srdivacky /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, 426199481Srdivacky /// this method does not allocate memory in the current output buffer, 427199481Srdivacky /// because a global may live longer than the current function. 428199481Srdivacky virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment); 429199481Srdivacky 430199481Srdivacky virtual void addRelocation(const MachineRelocation &MR) { 431199481Srdivacky Relocations.push_back(MR); 432199481Srdivacky } 433199481Srdivacky 434199481Srdivacky virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { 435199481Srdivacky if (MBBLocations.size() <= (unsigned)MBB->getNumber()) 436199481Srdivacky MBBLocations.resize((MBB->getNumber()+1)*2); 437199481Srdivacky MBBLocations[MBB->getNumber()] = getCurrentPCValue(); 438210299Sed if (MBB->hasAddressTaken()) 439210299Sed TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(), 440210299Sed (void*)getCurrentPCValue()); 441202375Srdivacky DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" 442199481Srdivacky << (void*) getCurrentPCValue() << "]\n"); 443199481Srdivacky } 444199481Srdivacky 445199481Srdivacky virtual uintptr_t getConstantPoolEntryAddress(unsigned Entry) const; 446199481Srdivacky virtual uintptr_t getJumpTableEntryAddress(unsigned Entry) const; 447199481Srdivacky 448210299Sed virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const{ 449199481Srdivacky assert(MBBLocations.size() > (unsigned)MBB->getNumber() && 450199481Srdivacky MBBLocations[MBB->getNumber()] && "MBB not emitted!"); 451199481Srdivacky return MBBLocations[MBB->getNumber()]; 452199481Srdivacky } 453199481Srdivacky 454199481Srdivacky /// retryWithMoreMemory - Log a retry and deallocate all memory for the 455199481Srdivacky /// given function. Increase the minimum allocation size so that we get 456199481Srdivacky /// more memory next time. 457199481Srdivacky void retryWithMoreMemory(MachineFunction &F); 458199481Srdivacky 459199481Srdivacky /// deallocateMemForFunction - Deallocate all memory for the specified 460199481Srdivacky /// function body. 461199481Srdivacky void deallocateMemForFunction(const Function *F); 462199481Srdivacky 463199481Srdivacky virtual void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn); 464199481Srdivacky 465205218Srdivacky virtual void emitLabel(MCSymbol *Label) { 466205218Srdivacky LabelLocations[Label] = getCurrentPCValue(); 467199481Srdivacky } 468199481Srdivacky 469207618Srdivacky virtual DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() { 470207618Srdivacky return &LabelLocations; 471207618Srdivacky } 472207618Srdivacky 473205218Srdivacky virtual uintptr_t getLabelAddress(MCSymbol *Label) const { 474205218Srdivacky assert(LabelLocations.count(Label) && "Label not emitted!"); 475205218Srdivacky return LabelLocations.find(Label)->second; 476199481Srdivacky } 477199481Srdivacky 478199481Srdivacky virtual void setModuleInfo(MachineModuleInfo* Info) { 479199481Srdivacky MMI = Info; 480199481Srdivacky if (DE.get()) DE->setModuleInfo(Info); 481199481Srdivacky } 482199481Srdivacky 483199481Srdivacky void setMemoryExecutable() { 484199481Srdivacky MemMgr->setMemoryExecutable(); 485199481Srdivacky } 486199481Srdivacky 487199481Srdivacky JITMemoryManager *getMemMgr() const { return MemMgr; } 488199481Srdivacky 489199481Srdivacky private: 490199481Srdivacky void *getPointerToGlobal(GlobalValue *GV, void *Reference, 491199481Srdivacky bool MayNeedFarStub); 492199481Srdivacky void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference); 493199481Srdivacky unsigned addSizeOfGlobal(const GlobalVariable *GV, unsigned Size); 494204792Srdivacky unsigned addSizeOfGlobalsInConstantVal( 495204792Srdivacky const Constant *C, unsigned Size, 496204792Srdivacky SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 497204792Srdivacky SmallVectorImpl<const GlobalVariable*> &Worklist); 498204792Srdivacky unsigned addSizeOfGlobalsInInitializer( 499204792Srdivacky const Constant *Init, unsigned Size, 500204792Srdivacky SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 501204792Srdivacky SmallVectorImpl<const GlobalVariable*> &Worklist); 502199481Srdivacky unsigned GetSizeOfGlobalsInBytes(MachineFunction &MF); 503199481Srdivacky }; 504193323Sed} 505193323Sed 506198892Srdivackyvoid CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) { 507204792Srdivacky JRS->EraseAllCallSitesForPrelocked(F); 508198892Srdivacky} 509198892Srdivacky 510204792SrdivackyFunction *JITResolverState::EraseStub(const MutexGuard &locked, void *Stub) { 511204792Srdivacky CallSiteToFunctionMapTy::iterator C2F_I = 512204792Srdivacky CallSiteToFunctionMap.find(Stub); 513204792Srdivacky if (C2F_I == CallSiteToFunctionMap.end()) { 514204792Srdivacky // Not a stub. 515204792Srdivacky return NULL; 516204792Srdivacky } 517204792Srdivacky 518204792Srdivacky StubToResolverMap->UnregisterStubResolver(Stub); 519204792Srdivacky 520204792Srdivacky Function *const F = C2F_I->second; 521204792Srdivacky#ifndef NDEBUG 522204792Srdivacky void *RealStub = FunctionToLazyStubMap.lookup(F); 523204792Srdivacky assert(RealStub == Stub && 524204792Srdivacky "Call-site that wasn't a stub passed in to EraseStub"); 525204792Srdivacky#endif 526204792Srdivacky FunctionToLazyStubMap.erase(F); 527204792Srdivacky CallSiteToFunctionMap.erase(C2F_I); 528204792Srdivacky 529204792Srdivacky // Remove the stub from the function->call-sites map, and remove the whole 530204792Srdivacky // entry from the map if that was the last call site. 531204792Srdivacky FunctionToCallSitesMapTy::iterator F2C_I = FunctionToCallSitesMap.find(F); 532204792Srdivacky assert(F2C_I != FunctionToCallSitesMap.end() && 533204792Srdivacky "FunctionToCallSitesMap broken"); 534204792Srdivacky bool Erased = F2C_I->second.erase(Stub); 535204792Srdivacky (void)Erased; 536204792Srdivacky assert(Erased && "FunctionToCallSitesMap broken"); 537204792Srdivacky if (F2C_I->second.empty()) 538204792Srdivacky FunctionToCallSitesMap.erase(F2C_I); 539204792Srdivacky 540204792Srdivacky return F; 541204792Srdivacky} 542204792Srdivacky 543204792Srdivackyvoid JITResolverState::EraseAllCallSitesForPrelocked(Function *F) { 544204792Srdivacky FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); 545204792Srdivacky if (F2C == FunctionToCallSitesMap.end()) 546204792Srdivacky return; 547204792Srdivacky StubToResolverMapTy &S2RMap = *StubToResolverMap; 548204792Srdivacky for (SmallPtrSet<void*, 1>::const_iterator I = F2C->second.begin(), 549204792Srdivacky E = F2C->second.end(); I != E; ++I) { 550204792Srdivacky S2RMap.UnregisterStubResolver(*I); 551204792Srdivacky bool Erased = CallSiteToFunctionMap.erase(*I); 552204792Srdivacky (void)Erased; 553204792Srdivacky assert(Erased && "Missing call site->function mapping"); 554204792Srdivacky } 555204792Srdivacky FunctionToCallSitesMap.erase(F2C); 556204792Srdivacky} 557204792Srdivacky 558204792Srdivackyvoid JITResolverState::EraseAllCallSitesPrelocked() { 559204792Srdivacky StubToResolverMapTy &S2RMap = *StubToResolverMap; 560204792Srdivacky for (CallSiteToFunctionMapTy::const_iterator 561204792Srdivacky I = CallSiteToFunctionMap.begin(), 562204792Srdivacky E = CallSiteToFunctionMap.end(); I != E; ++I) { 563204792Srdivacky S2RMap.UnregisterStubResolver(I->first); 564204792Srdivacky } 565204792Srdivacky CallSiteToFunctionMap.clear(); 566204792Srdivacky FunctionToCallSitesMap.clear(); 567204792Srdivacky} 568204792Srdivacky 569204792SrdivackyJITResolver::~JITResolver() { 570204792Srdivacky // No need to lock because we're in the destructor, and state isn't shared. 571204792Srdivacky state.EraseAllCallSitesPrelocked(); 572204792Srdivacky assert(!StubToResolverMap->ResolverHasStubs(this) && 573204792Srdivacky "Resolver destroyed with stubs still alive."); 574204792Srdivacky} 575204792Srdivacky 576199989Srdivacky/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub 577193323Sed/// if it has already been created. 578199989Srdivackyvoid *JITResolver::getLazyFunctionStubIfAvailable(Function *F) { 579193323Sed MutexGuard locked(TheJIT->lock); 580193323Sed 581193323Sed // If we already have a stub for this function, recycle it. 582199989Srdivacky return state.getFunctionToLazyStubMap(locked).lookup(F); 583193323Sed} 584193323Sed 585193323Sed/// getFunctionStub - This returns a pointer to a function stub, creating 586193323Sed/// one on demand as needed. 587199989Srdivackyvoid *JITResolver::getLazyFunctionStub(Function *F) { 588193323Sed MutexGuard locked(TheJIT->lock); 589193323Sed 590199989Srdivacky // If we already have a lazy stub for this function, recycle it. 591199989Srdivacky void *&Stub = state.getFunctionToLazyStubMap(locked)[F]; 592193323Sed if (Stub) return Stub; 593193323Sed 594198892Srdivacky // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we 595198892Srdivacky // must resolve the symbol now. 596198892Srdivacky void *Actual = TheJIT->isCompilingLazily() 597198892Srdivacky ? (void *)(intptr_t)LazyResolverFn : (void *)0; 598198892Srdivacky 599193323Sed // If this is an external declaration, attempt to resolve the address now 600193323Sed // to place in the stub. 601201360Srdivacky if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) { 602193323Sed Actual = TheJIT->getPointerToFunction(F); 603193323Sed 604193323Sed // If we resolved the symbol to a null address (eg. a weak external) 605199481Srdivacky // don't emit a stub. Return a null pointer to the application. 606199481Srdivacky if (!Actual) return 0; 607193323Sed } 608193323Sed 609199989Srdivacky TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); 610201360Srdivacky JE.startGVStub(F, SL.Size, SL.Alignment); 611193323Sed // Codegen a new stub, calling the lazy resolver or the actual address of the 612193323Sed // external function, if it was resolved. 613199481Srdivacky Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE); 614201360Srdivacky JE.finishGVStub(); 615193323Sed 616193323Sed if (Actual != (void*)(intptr_t)LazyResolverFn) { 617193323Sed // If we are getting the stub for an external function, we really want the 618193323Sed // address of the stub in the GlobalAddressMap for the JIT, not the address 619193323Sed // of the external function. 620193323Sed TheJIT->updateGlobalMapping(F, Stub); 621193323Sed } 622193323Sed 623202375Srdivacky DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '" 624198090Srdivacky << F->getName() << "'\n"); 625193323Sed 626204792Srdivacky if (TheJIT->isCompilingLazily()) { 627204792Srdivacky // Register this JITResolver as the one corresponding to this call site so 628204792Srdivacky // JITCompilerFn will be able to find it. 629204792Srdivacky StubToResolverMap->RegisterStubResolver(Stub, this); 630203954Srdivacky 631204792Srdivacky // Finally, keep track of the stub-to-Function mapping so that the 632204792Srdivacky // JITCompilerFn knows which function to compile! 633204792Srdivacky state.AddCallSite(locked, Stub, F); 634204792Srdivacky } else if (!Actual) { 635204792Srdivacky // If we are JIT'ing non-lazily but need to call a function that does not 636204792Srdivacky // exist yet, add it to the JIT's work list so that we can fill in the 637204792Srdivacky // stub address later. 638204792Srdivacky assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() && 639204792Srdivacky "'Actual' should have been set above."); 640204792Srdivacky TheJIT->addPendingFunction(F); 641204792Srdivacky } 642198090Srdivacky 643193323Sed return Stub; 644193323Sed} 645193323Sed 646193323Sed/// getGlobalValueIndirectSym - Return a lazy pointer containing the specified 647193323Sed/// GV address. 648193323Sedvoid *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { 649193323Sed MutexGuard locked(TheJIT->lock); 650193323Sed 651193323Sed // If we already have a stub for this global variable, recycle it. 652193323Sed void *&IndirectSym = state.getGlobalToIndirectSymMap(locked)[GV]; 653193323Sed if (IndirectSym) return IndirectSym; 654193323Sed 655193323Sed // Otherwise, codegen a new indirect symbol. 656193323Sed IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, 657199481Srdivacky JE); 658193323Sed 659202375Srdivacky DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym 660198090Srdivacky << "] for GV '" << GV->getName() << "'\n"); 661193323Sed 662193323Sed return IndirectSym; 663193323Sed} 664193323Sed 665193323Sed/// getExternalFunctionStub - Return a stub for the function at the 666193323Sed/// specified address, created lazily on demand. 667193323Sedvoid *JITResolver::getExternalFunctionStub(void *FnAddr) { 668193323Sed // If we already have a stub for this function, recycle it. 669193323Sed void *&Stub = ExternalFnToStubMap[FnAddr]; 670193323Sed if (Stub) return Stub; 671193323Sed 672199989Srdivacky TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); 673201360Srdivacky JE.startGVStub(0, SL.Size, SL.Alignment); 674199481Srdivacky Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr, JE); 675201360Srdivacky JE.finishGVStub(); 676193323Sed 677202375Srdivacky DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub 678198090Srdivacky << "] for external function at '" << FnAddr << "'\n"); 679193323Sed return Stub; 680193323Sed} 681193323Sed 682193323Sedunsigned JITResolver::getGOTIndexForAddr(void* addr) { 683193323Sed unsigned idx = revGOTMap[addr]; 684193323Sed if (!idx) { 685193323Sed idx = ++nextGOTIndex; 686193323Sed revGOTMap[addr] = idx; 687202375Srdivacky DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr [" 688198090Srdivacky << addr << "]\n"); 689193323Sed } 690193323Sed return idx; 691193323Sed} 692193323Sed 693193323Sedvoid JITResolver::getRelocatableGVs(SmallVectorImpl<GlobalValue*> &GVs, 694193323Sed SmallVectorImpl<void*> &Ptrs) { 695193323Sed MutexGuard locked(TheJIT->lock); 696199481Srdivacky 697199989Srdivacky const FunctionToLazyStubMapTy &FM = state.getFunctionToLazyStubMap(locked); 698193323Sed GlobalToIndirectSymMapTy &GM = state.getGlobalToIndirectSymMap(locked); 699199481Srdivacky 700199989Srdivacky for (FunctionToLazyStubMapTy::const_iterator i = FM.begin(), e = FM.end(); 701198396Srdivacky i != e; ++i){ 702193323Sed Function *F = i->first; 703193323Sed if (F->isDeclaration() && F->hasExternalLinkage()) { 704193323Sed GVs.push_back(i->first); 705193323Sed Ptrs.push_back(i->second); 706193323Sed } 707193323Sed } 708193323Sed for (GlobalToIndirectSymMapTy::iterator i = GM.begin(), e = GM.end(); 709193323Sed i != e; ++i) { 710193323Sed GVs.push_back(i->first); 711193323Sed Ptrs.push_back(i->second); 712193323Sed } 713193323Sed} 714193323Sed 715193323Sed/// JITCompilerFn - This function is called when a lazy compilation stub has 716193323Sed/// been entered. It looks up which function this stub corresponds to, compiles 717193323Sed/// it if necessary, then returns the resultant function pointer. 718193323Sedvoid *JITResolver::JITCompilerFn(void *Stub) { 719203954Srdivacky JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub); 720203954Srdivacky assert(JR && "Unable to find the corresponding JITResolver to the call site"); 721199481Srdivacky 722193323Sed Function* F = 0; 723193323Sed void* ActualPtr = 0; 724193323Sed 725193323Sed { 726193323Sed // Only lock for getting the Function. The call getPointerToFunction made 727193323Sed // in this function might trigger function materializing, which requires 728193323Sed // JIT lock to be unlocked. 729203954Srdivacky MutexGuard locked(JR->TheJIT->lock); 730193323Sed 731198396Srdivacky // The address given to us for the stub may not be exactly right, it might 732198396Srdivacky // be a little bit after the stub. As such, use upper_bound to find it. 733198396Srdivacky pair<void*, Function*> I = 734203954Srdivacky JR->state.LookupFunctionFromCallSite(locked, Stub); 735198396Srdivacky F = I.second; 736198396Srdivacky ActualPtr = I.first; 737193323Sed } 738193323Sed 739193323Sed // If we have already code generated the function, just return the address. 740203954Srdivacky void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F); 741199481Srdivacky 742193323Sed if (!Result) { 743193323Sed // Otherwise we don't have it, do lazy compilation now. 744199481Srdivacky 745193323Sed // If lazy compilation is disabled, emit a useful error message and abort. 746203954Srdivacky if (!JR->TheJIT->isCompilingLazily()) { 747207618Srdivacky report_fatal_error("LLVM JIT requested to do lazy compilation of function '" 748198090Srdivacky + F->getName() + "' when lazy compiles are disabled!"); 749193323Sed } 750199481Srdivacky 751202375Srdivacky DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName() 752198090Srdivacky << "' In stub ptr = " << Stub << " actual ptr = " 753198090Srdivacky << ActualPtr << "\n"); 754193323Sed 755203954Srdivacky Result = JR->TheJIT->getPointerToFunction(F); 756193323Sed } 757198396Srdivacky 758198396Srdivacky // Reacquire the lock to update the GOT map. 759203954Srdivacky MutexGuard locked(JR->TheJIT->lock); 760193323Sed 761198396Srdivacky // We might like to remove the call site from the CallSiteToFunction map, but 762198396Srdivacky // we can't do that! Multiple threads could be stuck, waiting to acquire the 763198396Srdivacky // lock above. As soon as the 1st function finishes compiling the function, 764198396Srdivacky // the next one will be released, and needs to be able to find the function it 765198396Srdivacky // needs to call. 766193323Sed 767193323Sed // FIXME: We could rewrite all references to this stub if we knew them. 768193323Sed 769193323Sed // What we will do is set the compiled function address to map to the 770193323Sed // same GOT entry as the stub so that later clients may update the GOT 771193323Sed // if they see it still using the stub address. 772193323Sed // Note: this is done so the Resolver doesn't have to manage GOT memory 773193323Sed // Do this without allocating map space if the target isn't using a GOT 774203954Srdivacky if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end()) 775203954Srdivacky JR->revGOTMap[Result] = JR->revGOTMap[Stub]; 776193323Sed 777193323Sed return Result; 778193323Sed} 779193323Sed 780193323Sed//===----------------------------------------------------------------------===// 781193323Sed// JITEmitter code. 782193323Sed// 783193323Sedvoid *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, 784199481Srdivacky bool MayNeedFarStub) { 785193323Sed if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) 786193323Sed return TheJIT->getOrEmitGlobalVariable(GV); 787193323Sed 788193323Sed if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 789193323Sed return TheJIT->getPointerToGlobal(GA->resolveAliasedGlobal(false)); 790193323Sed 791193323Sed // If we have already compiled the function, return a pointer to its body. 792193323Sed Function *F = cast<Function>(V); 793199481Srdivacky 794199989Srdivacky void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F); 795199481Srdivacky if (FnStub) { 796199989Srdivacky // Return the function stub if it's already created. We do this first so 797199989Srdivacky // that we're returning the same address for the function as any previous 798199989Srdivacky // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be 799199989Srdivacky // close enough to call. 800199481Srdivacky return FnStub; 801193323Sed } 802199481Srdivacky 803199989Srdivacky // If we know the target can handle arbitrary-distance calls, try to 804199989Srdivacky // return a direct pointer. 805199989Srdivacky if (!MayNeedFarStub) { 806199989Srdivacky // If we have code, go ahead and return that. 807199989Srdivacky void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); 808199989Srdivacky if (ResultPtr) return ResultPtr; 809193323Sed 810199989Srdivacky // If this is an external function pointer, we can force the JIT to 811199989Srdivacky // 'compile' it, which really just adds it to the map. 812201360Srdivacky if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) 813199989Srdivacky return TheJIT->getPointerToFunction(F); 814199989Srdivacky } 815193323Sed 816204792Srdivacky // Otherwise, we may need a to emit a stub, and, conservatively, we always do 817204792Srdivacky // so. Note that it's possible to return null from getLazyFunctionStub in the 818204792Srdivacky // case of a weak extern that fails to resolve. 819204792Srdivacky return Resolver.getLazyFunctionStub(F); 820193323Sed} 821193323Sed 822199481Srdivackyvoid *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { 823193323Sed // Make sure GV is emitted first, and create a stub containing the fully 824193323Sed // resolved address. 825199481Srdivacky void *GVAddress = getPointerToGlobal(V, Reference, false); 826193323Sed void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); 827193323Sed return StubAddr; 828193323Sed} 829193323Sed 830198090Srdivackyvoid JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { 831206124Srdivacky if (DL.isUnknown()) return; 832206124Srdivacky if (!BeforePrintingInsn) return; 833207618Srdivacky 834207618Srdivacky const LLVMContext& Context = EmissionDetails.MF->getFunction()->getContext(); 835198090Srdivacky 836207618Srdivacky if (DL.getScope(Context) != 0 && PrevDL != DL) { 837206124Srdivacky JITEvent_EmittedFunctionDetails::LineStart NextLine; 838206124Srdivacky NextLine.Address = getCurrentPCValue(); 839206124Srdivacky NextLine.Loc = DL; 840206124Srdivacky EmissionDetails.LineStarts.push_back(NextLine); 841206124Srdivacky } 842199481Srdivacky 843207618Srdivacky PrevDL = DL; 844198090Srdivacky} 845198090Srdivacky 846193323Sedstatic unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, 847193323Sed const TargetData *TD) { 848193323Sed const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 849193323Sed if (Constants.empty()) return 0; 850193323Sed 851193323Sed unsigned Size = 0; 852193323Sed for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 853193323Sed MachineConstantPoolEntry CPE = Constants[i]; 854193323Sed unsigned AlignMask = CPE.getAlignment() - 1; 855193323Sed Size = (Size + AlignMask) & ~AlignMask; 856193323Sed const Type *Ty = CPE.getType(); 857193323Sed Size += TD->getTypeAllocSize(Ty); 858193323Sed } 859193323Sed return Size; 860193323Sed} 861193323Sed 862203954Srdivackystatic unsigned GetJumpTableSizeInBytes(MachineJumpTableInfo *MJTI, JIT *jit) { 863193323Sed const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 864193323Sed if (JT.empty()) return 0; 865199481Srdivacky 866193323Sed unsigned NumEntries = 0; 867193323Sed for (unsigned i = 0, e = JT.size(); i != e; ++i) 868193323Sed NumEntries += JT[i].MBBs.size(); 869193323Sed 870203954Srdivacky return NumEntries * MJTI->getEntrySize(*jit->getTargetData()); 871193323Sed} 872193323Sed 873193323Sedstatic uintptr_t RoundUpToAlign(uintptr_t Size, unsigned Alignment) { 874193323Sed if (Alignment == 0) Alignment = 1; 875199481Srdivacky // Since we do not know where the buffer will be allocated, be pessimistic. 876193323Sed return Size + Alignment; 877193323Sed} 878193323Sed 879193323Sed/// addSizeOfGlobal - add the size of the global (plus any alignment padding) 880193323Sed/// into the running total Size. 881193323Sed 882193323Sedunsigned JITEmitter::addSizeOfGlobal(const GlobalVariable *GV, unsigned Size) { 883193323Sed const Type *ElTy = GV->getType()->getElementType(); 884193323Sed size_t GVSize = (size_t)TheJIT->getTargetData()->getTypeAllocSize(ElTy); 885199481Srdivacky size_t GVAlign = 886193323Sed (size_t)TheJIT->getTargetData()->getPreferredAlignment(GV); 887202375Srdivacky DEBUG(dbgs() << "JIT: Adding in size " << GVSize << " alignment " << GVAlign); 888193323Sed DEBUG(GV->dump()); 889193323Sed // Assume code section ends with worst possible alignment, so first 890193323Sed // variable needs maximal padding. 891193323Sed if (Size==0) 892193323Sed Size = 1; 893193323Sed Size = ((Size+GVAlign-1)/GVAlign)*GVAlign; 894193323Sed Size += GVSize; 895193323Sed return Size; 896193323Sed} 897193323Sed 898193323Sed/// addSizeOfGlobalsInConstantVal - find any globals that we haven't seen yet 899204792Srdivacky/// but are referenced from the constant; put them in SeenGlobals and the 900204792Srdivacky/// Worklist, and add their size into the running total Size. 901193323Sed 902204792Srdivackyunsigned JITEmitter::addSizeOfGlobalsInConstantVal( 903204792Srdivacky const Constant *C, 904204792Srdivacky unsigned Size, 905204792Srdivacky SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 906204792Srdivacky SmallVectorImpl<const GlobalVariable*> &Worklist) { 907193323Sed // If its undefined, return the garbage. 908193323Sed if (isa<UndefValue>(C)) 909193323Sed return Size; 910193323Sed 911193323Sed // If the value is a ConstantExpr 912193323Sed if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { 913193323Sed Constant *Op0 = CE->getOperand(0); 914193323Sed switch (CE->getOpcode()) { 915193323Sed case Instruction::GetElementPtr: 916193323Sed case Instruction::Trunc: 917193323Sed case Instruction::ZExt: 918193323Sed case Instruction::SExt: 919193323Sed case Instruction::FPTrunc: 920193323Sed case Instruction::FPExt: 921193323Sed case Instruction::UIToFP: 922193323Sed case Instruction::SIToFP: 923193323Sed case Instruction::FPToUI: 924193323Sed case Instruction::FPToSI: 925193323Sed case Instruction::PtrToInt: 926193323Sed case Instruction::IntToPtr: 927193323Sed case Instruction::BitCast: { 928204792Srdivacky Size = addSizeOfGlobalsInConstantVal(Op0, Size, SeenGlobals, Worklist); 929193323Sed break; 930193323Sed } 931193323Sed case Instruction::Add: 932193574Sed case Instruction::FAdd: 933193323Sed case Instruction::Sub: 934193574Sed case Instruction::FSub: 935193323Sed case Instruction::Mul: 936193574Sed case Instruction::FMul: 937193323Sed case Instruction::UDiv: 938193323Sed case Instruction::SDiv: 939193323Sed case Instruction::URem: 940193323Sed case Instruction::SRem: 941193323Sed case Instruction::And: 942193323Sed case Instruction::Or: 943193323Sed case Instruction::Xor: { 944204792Srdivacky Size = addSizeOfGlobalsInConstantVal(Op0, Size, SeenGlobals, Worklist); 945204792Srdivacky Size = addSizeOfGlobalsInConstantVal(CE->getOperand(1), Size, 946204792Srdivacky SeenGlobals, Worklist); 947193323Sed break; 948193323Sed } 949193323Sed default: { 950198090Srdivacky std::string msg; 951198090Srdivacky raw_string_ostream Msg(msg); 952198090Srdivacky Msg << "ConstantExpr not handled: " << *CE; 953207618Srdivacky report_fatal_error(Msg.str()); 954193323Sed } 955193323Sed } 956193323Sed } 957193323Sed 958193323Sed if (C->getType()->getTypeID() == Type::PointerTyID) 959193323Sed if (const GlobalVariable* GV = dyn_cast<GlobalVariable>(C)) 960204792Srdivacky if (SeenGlobals.insert(GV)) { 961204792Srdivacky Worklist.push_back(GV); 962193323Sed Size = addSizeOfGlobal(GV, Size); 963204792Srdivacky } 964193323Sed 965193323Sed return Size; 966193323Sed} 967193323Sed 968193323Sed/// addSizeOfGLobalsInInitializer - handle any globals that we haven't seen yet 969193323Sed/// but are referenced from the given initializer. 970193323Sed 971204792Srdivackyunsigned JITEmitter::addSizeOfGlobalsInInitializer( 972204792Srdivacky const Constant *Init, 973204792Srdivacky unsigned Size, 974204792Srdivacky SmallPtrSet<const GlobalVariable*, 8> &SeenGlobals, 975204792Srdivacky SmallVectorImpl<const GlobalVariable*> &Worklist) { 976193323Sed if (!isa<UndefValue>(Init) && 977193323Sed !isa<ConstantVector>(Init) && 978193323Sed !isa<ConstantAggregateZero>(Init) && 979193323Sed !isa<ConstantArray>(Init) && 980193323Sed !isa<ConstantStruct>(Init) && 981193323Sed Init->getType()->isFirstClassType()) 982204792Srdivacky Size = addSizeOfGlobalsInConstantVal(Init, Size, SeenGlobals, Worklist); 983193323Sed return Size; 984193323Sed} 985193323Sed 986193323Sed/// GetSizeOfGlobalsInBytes - walk the code for the function, looking for 987193323Sed/// globals; then walk the initializers of those globals looking for more. 988193323Sed/// If their size has not been considered yet, add it into the running total 989193323Sed/// Size. 990193323Sed 991193323Sedunsigned JITEmitter::GetSizeOfGlobalsInBytes(MachineFunction &MF) { 992193323Sed unsigned Size = 0; 993204792Srdivacky SmallPtrSet<const GlobalVariable*, 8> SeenGlobals; 994193323Sed 995199481Srdivacky for (MachineFunction::iterator MBB = MF.begin(), E = MF.end(); 996193323Sed MBB != E; ++MBB) { 997193323Sed for (MachineBasicBlock::const_iterator I = MBB->begin(), E = MBB->end(); 998193323Sed I != E; ++I) { 999193323Sed const TargetInstrDesc &Desc = I->getDesc(); 1000193323Sed const MachineInstr &MI = *I; 1001193323Sed unsigned NumOps = Desc.getNumOperands(); 1002193323Sed for (unsigned CurOp = 0; CurOp < NumOps; CurOp++) { 1003193323Sed const MachineOperand &MO = MI.getOperand(CurOp); 1004193323Sed if (MO.isGlobal()) { 1005207618Srdivacky const GlobalValue* V = MO.getGlobal(); 1006193323Sed const GlobalVariable *GV = dyn_cast<const GlobalVariable>(V); 1007193323Sed if (!GV) 1008193323Sed continue; 1009193323Sed // If seen in previous function, it will have an entry here. 1010207618Srdivacky if (TheJIT->getPointerToGlobalIfAvailable( 1011207618Srdivacky const_cast<GlobalVariable *>(GV))) 1012193323Sed continue; 1013193323Sed // If seen earlier in this function, it will have an entry here. 1014193323Sed // FIXME: it should be possible to combine these tables, by 1015193323Sed // assuming the addresses of the new globals in this module 1016193323Sed // start at 0 (or something) and adjusting them after codegen 1017193323Sed // complete. Another possibility is to grab a marker bit in GV. 1018204792Srdivacky if (SeenGlobals.insert(GV)) 1019193323Sed // A variable as yet unseen. Add in its size. 1020193323Sed Size = addSizeOfGlobal(GV, Size); 1021193323Sed } 1022193323Sed } 1023193323Sed } 1024193323Sed } 1025202375Srdivacky DEBUG(dbgs() << "JIT: About to look through initializers\n"); 1026193323Sed // Look for more globals that are referenced only from initializers. 1027204792Srdivacky SmallVector<const GlobalVariable*, 8> Worklist( 1028204792Srdivacky SeenGlobals.begin(), SeenGlobals.end()); 1029204792Srdivacky while (!Worklist.empty()) { 1030204792Srdivacky const GlobalVariable* GV = Worklist.back(); 1031204792Srdivacky Worklist.pop_back(); 1032193323Sed if (GV->hasInitializer()) 1033204792Srdivacky Size = addSizeOfGlobalsInInitializer(GV->getInitializer(), Size, 1034204792Srdivacky SeenGlobals, Worklist); 1035193323Sed } 1036193323Sed 1037193323Sed return Size; 1038193323Sed} 1039193323Sed 1040193323Sedvoid JITEmitter::startFunction(MachineFunction &F) { 1041202375Srdivacky DEBUG(dbgs() << "JIT: Starting CodeGen of Function " 1042198090Srdivacky << F.getFunction()->getName() << "\n"); 1043193323Sed 1044193323Sed uintptr_t ActualSize = 0; 1045193323Sed // Set the memory writable, if it's not already 1046193323Sed MemMgr->setMemoryWritable(); 1047193323Sed if (MemMgr->NeedsExactSize()) { 1048202375Srdivacky DEBUG(dbgs() << "JIT: ExactSize\n"); 1049193323Sed const TargetInstrInfo* TII = F.getTarget().getInstrInfo(); 1050193323Sed MachineConstantPool *MCP = F.getConstantPool(); 1051199481Srdivacky 1052193323Sed // Ensure the constant pool/jump table info is at least 4-byte aligned. 1053193323Sed ActualSize = RoundUpToAlign(ActualSize, 16); 1054199481Srdivacky 1055193323Sed // Add the alignment of the constant pool 1056193323Sed ActualSize = RoundUpToAlign(ActualSize, MCP->getConstantPoolAlignment()); 1057193323Sed 1058193323Sed // Add the constant pool size 1059193323Sed ActualSize += GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); 1060193323Sed 1061203954Srdivacky if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) { 1062203954Srdivacky // Add the aligment of the jump table info 1063203954Srdivacky ActualSize = RoundUpToAlign(ActualSize, 1064203954Srdivacky MJTI->getEntryAlignment(*TheJIT->getTargetData())); 1065193323Sed 1066203954Srdivacky // Add the jump table size 1067203954Srdivacky ActualSize += GetJumpTableSizeInBytes(MJTI, TheJIT); 1068203954Srdivacky } 1069199481Srdivacky 1070193323Sed // Add the alignment for the function 1071193323Sed ActualSize = RoundUpToAlign(ActualSize, 1072193323Sed std::max(F.getFunction()->getAlignment(), 8U)); 1073193323Sed 1074193323Sed // Add the function size 1075193323Sed ActualSize += TII->GetFunctionSizeInBytes(F); 1076193323Sed 1077202375Srdivacky DEBUG(dbgs() << "JIT: ActualSize before globals " << ActualSize << "\n"); 1078193323Sed // Add the size of the globals that will be allocated after this function. 1079193323Sed // These are all the ones referenced from this function that were not 1080193323Sed // previously allocated. 1081193323Sed ActualSize += GetSizeOfGlobalsInBytes(F); 1082202375Srdivacky DEBUG(dbgs() << "JIT: ActualSize after globals " << ActualSize << "\n"); 1083198090Srdivacky } else if (SizeEstimate > 0) { 1084198090Srdivacky // SizeEstimate will be non-zero on reallocation attempts. 1085198090Srdivacky ActualSize = SizeEstimate; 1086193323Sed } 1087193323Sed 1088193323Sed BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), 1089193323Sed ActualSize); 1090193323Sed BufferEnd = BufferBegin+ActualSize; 1091198396Srdivacky EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin; 1092198396Srdivacky 1093193323Sed // Ensure the constant pool/jump table info is at least 4-byte aligned. 1094193323Sed emitAlignment(16); 1095193323Sed 1096193323Sed emitConstantPool(F.getConstantPool()); 1097203954Srdivacky if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) 1098203954Srdivacky initJumpTableInfo(MJTI); 1099193323Sed 1100193323Sed // About to start emitting the machine code for the function. 1101193323Sed emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); 1102193323Sed TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); 1103198892Srdivacky EmittedFunctions[F.getFunction()].Code = CurBufferPtr; 1104193323Sed 1105193323Sed MBBLocations.clear(); 1106198090Srdivacky 1107198090Srdivacky EmissionDetails.MF = &F; 1108198090Srdivacky EmissionDetails.LineStarts.clear(); 1109193323Sed} 1110193323Sed 1111193323Sedbool JITEmitter::finishFunction(MachineFunction &F) { 1112193323Sed if (CurBufferPtr == BufferEnd) { 1113198090Srdivacky // We must call endFunctionBody before retrying, because 1114198090Srdivacky // deallocateMemForFunction requires it. 1115198090Srdivacky MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 1116198090Srdivacky retryWithMoreMemory(F); 1117198090Srdivacky return true; 1118193323Sed } 1119198090Srdivacky 1120203954Srdivacky if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) 1121203954Srdivacky emitJumpTableInfo(MJTI); 1122198090Srdivacky 1123193323Sed // FnStart is the start of the text, not the start of the constant pool and 1124193323Sed // other per-function data. 1125193574Sed uint8_t *FnStart = 1126193574Sed (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); 1127193323Sed 1128193323Sed // FnEnd is the end of the function's machine code. 1129193574Sed uint8_t *FnEnd = CurBufferPtr; 1130193323Sed 1131193323Sed if (!Relocations.empty()) { 1132193323Sed CurFn = F.getFunction(); 1133193323Sed NumRelos += Relocations.size(); 1134193323Sed 1135193323Sed // Resolve the relocations to concrete pointers. 1136193323Sed for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { 1137193323Sed MachineRelocation &MR = Relocations[i]; 1138193323Sed void *ResultPtr = 0; 1139193323Sed if (!MR.letTargetResolve()) { 1140193323Sed if (MR.isExternalSymbol()) { 1141193323Sed ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), 1142193323Sed false); 1143202375Srdivacky DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" 1144199481Srdivacky << ResultPtr << "]\n"); 1145193323Sed 1146193323Sed // If the target REALLY wants a stub for this function, emit it now. 1147199481Srdivacky if (MR.mayNeedFarStub()) { 1148199481Srdivacky ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); 1149193323Sed } 1150193323Sed } else if (MR.isGlobalValue()) { 1151193323Sed ResultPtr = getPointerToGlobal(MR.getGlobalValue(), 1152193323Sed BufferBegin+MR.getMachineCodeOffset(), 1153199481Srdivacky MR.mayNeedFarStub()); 1154193323Sed } else if (MR.isIndirectSymbol()) { 1155199481Srdivacky ResultPtr = getPointerToGVIndirectSym( 1156199481Srdivacky MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset()); 1157193323Sed } else if (MR.isBasicBlock()) { 1158193323Sed ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); 1159193323Sed } else if (MR.isConstantPoolIndex()) { 1160193323Sed ResultPtr = (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); 1161193323Sed } else { 1162193323Sed assert(MR.isJumpTableIndex()); 1163193323Sed ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); 1164193323Sed } 1165193323Sed 1166193323Sed MR.setResultPointer(ResultPtr); 1167193323Sed } 1168193323Sed 1169193323Sed // if we are managing the GOT and the relocation wants an index, 1170193323Sed // give it one 1171193323Sed if (MR.isGOTRelative() && MemMgr->isManagingGOT()) { 1172193323Sed unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); 1173193323Sed MR.setGOTIndex(idx); 1174193323Sed if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { 1175202375Srdivacky DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr 1176198090Srdivacky << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 1177198090Srdivacky << "\n"); 1178193323Sed ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; 1179193323Sed } 1180193323Sed } 1181193323Sed } 1182193323Sed 1183193323Sed CurFn = 0; 1184193323Sed TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], 1185193323Sed Relocations.size(), MemMgr->getGOTBase()); 1186193323Sed } 1187193323Sed 1188193323Sed // Update the GOT entry for F to point to the new code. 1189193323Sed if (MemMgr->isManagingGOT()) { 1190193323Sed unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); 1191193323Sed if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { 1192202375Srdivacky DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin 1193198090Srdivacky << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 1194198090Srdivacky << "\n"); 1195193323Sed ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; 1196193323Sed } 1197193323Sed } 1198193323Sed 1199193323Sed // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for 1200193323Sed // global variables that were referenced in the relocations. 1201193323Sed MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 1202193323Sed 1203193323Sed if (CurBufferPtr == BufferEnd) { 1204198090Srdivacky retryWithMoreMemory(F); 1205198090Srdivacky return true; 1206198090Srdivacky } else { 1207198090Srdivacky // Now that we've succeeded in emitting the function, reset the 1208198090Srdivacky // SizeEstimate back down to zero. 1209198090Srdivacky SizeEstimate = 0; 1210193323Sed } 1211193323Sed 1212193323Sed BufferBegin = CurBufferPtr = 0; 1213193323Sed NumBytes += FnEnd-FnStart; 1214193323Sed 1215193323Sed // Invalidate the icache if necessary. 1216193323Sed sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart); 1217193323Sed 1218195098Sed TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, 1219198090Srdivacky EmissionDetails); 1220195098Sed 1221207618Srdivacky // Reset the previous debug location. 1222207618Srdivacky PrevDL = DebugLoc(); 1223207618Srdivacky 1224202375Srdivacky DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart 1225198090Srdivacky << "] Function: " << F.getFunction()->getName() 1226198090Srdivacky << ": " << (FnEnd-FnStart) << " bytes of text, " 1227198090Srdivacky << Relocations.size() << " relocations\n"); 1228193323Sed 1229193323Sed Relocations.clear(); 1230193323Sed ConstPoolAddresses.clear(); 1231193323Sed 1232193323Sed // Mark code region readable and executable if it's not so already. 1233193323Sed MemMgr->setMemoryExecutable(); 1234193323Sed 1235207618Srdivacky DEBUG({ 1236207618Srdivacky if (sys::hasDisassembler()) { 1237207618Srdivacky dbgs() << "JIT: Disassembled code:\n"; 1238207618Srdivacky dbgs() << sys::disassembleBuffer(FnStart, FnEnd-FnStart, 1239207618Srdivacky (uintptr_t)FnStart); 1240207618Srdivacky } else { 1241207618Srdivacky dbgs() << "JIT: Binary code:\n"; 1242207618Srdivacky uint8_t* q = FnStart; 1243207618Srdivacky for (int i = 0; q < FnEnd; q += 4, ++i) { 1244207618Srdivacky if (i == 4) 1245207618Srdivacky i = 0; 1246207618Srdivacky if (i == 0) 1247207618Srdivacky dbgs() << "JIT: " << (long)(q - FnStart) << ": "; 1248207618Srdivacky bool Done = false; 1249207618Srdivacky for (int j = 3; j >= 0; --j) { 1250207618Srdivacky if (q + j >= FnEnd) 1251207618Srdivacky Done = true; 1252207618Srdivacky else 1253207618Srdivacky dbgs() << (unsigned short)q[j]; 1254207618Srdivacky } 1255207618Srdivacky if (Done) 1256207618Srdivacky break; 1257207618Srdivacky dbgs() << ' '; 1258207618Srdivacky if (i == 3) 1259207618Srdivacky dbgs() << '\n'; 1260193323Sed } 1261207618Srdivacky dbgs()<< '\n'; 1262193323Sed } 1263207618Srdivacky }); 1264198090Srdivacky 1265207618Srdivacky if (JITExceptionHandling || JITEmitDebugInfo) { 1266193323Sed uintptr_t ActualSize = 0; 1267201360Srdivacky SavedBufferBegin = BufferBegin; 1268201360Srdivacky SavedBufferEnd = BufferEnd; 1269201360Srdivacky SavedCurBufferPtr = CurBufferPtr; 1270198090Srdivacky 1271207618Srdivacky if (MemMgr->NeedsExactSize()) 1272193323Sed ActualSize = DE->GetDwarfTableSizeInBytes(F, *this, FnStart, FnEnd); 1273193323Sed 1274193323Sed BufferBegin = CurBufferPtr = MemMgr->startExceptionTable(F.getFunction(), 1275193323Sed ActualSize); 1276193323Sed BufferEnd = BufferBegin+ActualSize; 1277198396Srdivacky EmittedFunctions[F.getFunction()].ExceptionTable = BufferBegin; 1278198090Srdivacky uint8_t *EhStart; 1279198090Srdivacky uint8_t *FrameRegister = DE->EmitDwarfTable(F, *this, FnStart, FnEnd, 1280198090Srdivacky EhStart); 1281193323Sed MemMgr->endExceptionTable(F.getFunction(), BufferBegin, CurBufferPtr, 1282193323Sed FrameRegister); 1283198090Srdivacky uint8_t *EhEnd = CurBufferPtr; 1284201360Srdivacky BufferBegin = SavedBufferBegin; 1285201360Srdivacky BufferEnd = SavedBufferEnd; 1286201360Srdivacky CurBufferPtr = SavedCurBufferPtr; 1287193323Sed 1288207618Srdivacky if (JITExceptionHandling) { 1289198090Srdivacky TheJIT->RegisterTable(FrameRegister); 1290198090Srdivacky } 1291198090Srdivacky 1292198090Srdivacky if (JITEmitDebugInfo) { 1293198090Srdivacky DebugInfo I; 1294198090Srdivacky I.FnStart = FnStart; 1295198090Srdivacky I.FnEnd = FnEnd; 1296198090Srdivacky I.EhStart = EhStart; 1297198090Srdivacky I.EhEnd = EhEnd; 1298198090Srdivacky DR->RegisterFunction(F.getFunction(), I); 1299198090Srdivacky } 1300193323Sed } 1301193323Sed 1302193323Sed if (MMI) 1303193323Sed MMI->EndFunction(); 1304199481Srdivacky 1305193323Sed return false; 1306193323Sed} 1307193323Sed 1308198090Srdivackyvoid JITEmitter::retryWithMoreMemory(MachineFunction &F) { 1309202375Srdivacky DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n"); 1310198090Srdivacky Relocations.clear(); // Clear the old relocations or we'll reapply them. 1311198090Srdivacky ConstPoolAddresses.clear(); 1312198090Srdivacky ++NumRetries; 1313198090Srdivacky deallocateMemForFunction(F.getFunction()); 1314198090Srdivacky // Try again with at least twice as much free space. 1315198090Srdivacky SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); 1316210299Sed 1317210299Sed for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){ 1318210299Sed if (MBB->hasAddressTaken()) 1319210299Sed TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock()); 1320210299Sed } 1321198090Srdivacky} 1322198090Srdivacky 1323193323Sed/// deallocateMemForFunction - Deallocate all memory for the specified 1324193323Sed/// function body. Also drop any references the function has to stubs. 1325198892Srdivacky/// May be called while the Function is being destroyed inside ~Value(). 1326198090Srdivackyvoid JITEmitter::deallocateMemForFunction(const Function *F) { 1327198892Srdivacky ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator 1328198892Srdivacky Emitted = EmittedFunctions.find(F); 1329198396Srdivacky if (Emitted != EmittedFunctions.end()) { 1330198396Srdivacky MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody); 1331198396Srdivacky MemMgr->deallocateExceptionTable(Emitted->second.ExceptionTable); 1332198892Srdivacky TheJIT->NotifyFreeingMachineCode(Emitted->second.Code); 1333198892Srdivacky 1334198396Srdivacky EmittedFunctions.erase(Emitted); 1335198396Srdivacky } 1336193323Sed 1337198090Srdivacky // TODO: Do we need to unregister exception handling information from libgcc 1338198090Srdivacky // here? 1339198090Srdivacky 1340198090Srdivacky if (JITEmitDebugInfo) { 1341198090Srdivacky DR->UnregisterFunction(F); 1342198090Srdivacky } 1343193323Sed} 1344193323Sed 1345193323Sed 1346193323Sedvoid* JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { 1347193323Sed if (BufferBegin) 1348193323Sed return JITCodeEmitter::allocateSpace(Size, Alignment); 1349193323Sed 1350193323Sed // create a new memory block if there is no active one. 1351193323Sed // care must be taken so that BufferBegin is invalidated when a 1352193323Sed // block is trimmed 1353193323Sed BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment); 1354193323Sed BufferEnd = BufferBegin+Size; 1355193323Sed return CurBufferPtr; 1356193323Sed} 1357193323Sed 1358198090Srdivackyvoid* JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { 1359198090Srdivacky // Delegate this call through the memory manager. 1360198090Srdivacky return MemMgr->allocateGlobal(Size, Alignment); 1361198090Srdivacky} 1362198090Srdivacky 1363193323Sedvoid JITEmitter::emitConstantPool(MachineConstantPool *MCP) { 1364193323Sed if (TheJIT->getJITInfo().hasCustomConstantPool()) 1365193323Sed return; 1366193323Sed 1367193323Sed const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 1368193323Sed if (Constants.empty()) return; 1369193323Sed 1370193323Sed unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); 1371193323Sed unsigned Align = MCP->getConstantPoolAlignment(); 1372193323Sed ConstantPoolBase = allocateSpace(Size, Align); 1373193323Sed ConstantPool = MCP; 1374193323Sed 1375193323Sed if (ConstantPoolBase == 0) return; // Buffer overflow. 1376193323Sed 1377202375Srdivacky DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase 1378198090Srdivacky << "] (size: " << Size << ", alignment: " << Align << ")\n"); 1379193323Sed 1380193323Sed // Initialize the memory for all of the constant pool entries. 1381193323Sed unsigned Offset = 0; 1382193323Sed for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 1383193323Sed MachineConstantPoolEntry CPE = Constants[i]; 1384193323Sed unsigned AlignMask = CPE.getAlignment() - 1; 1385193323Sed Offset = (Offset + AlignMask) & ~AlignMask; 1386193323Sed 1387193323Sed uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset; 1388193323Sed ConstPoolAddresses.push_back(CAddr); 1389193323Sed if (CPE.isMachineConstantPoolEntry()) { 1390193323Sed // FIXME: add support to lower machine constant pool values into bytes! 1391207618Srdivacky report_fatal_error("Initialize memory with machine specific constant pool" 1392198090Srdivacky "entry has not been implemented!"); 1393193323Sed } 1394193323Sed TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); 1395202375Srdivacky DEBUG(dbgs() << "JIT: CP" << i << " at [0x"; 1396202375Srdivacky dbgs().write_hex(CAddr) << "]\n"); 1397193323Sed 1398193323Sed const Type *Ty = CPE.Val.ConstVal->getType(); 1399193323Sed Offset += TheJIT->getTargetData()->getTypeAllocSize(Ty); 1400193323Sed } 1401193323Sed} 1402193323Sed 1403193323Sedvoid JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { 1404193323Sed if (TheJIT->getJITInfo().hasCustomJumpTables()) 1405193323Sed return; 1406205218Srdivacky if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) 1407205218Srdivacky return; 1408193323Sed 1409193323Sed const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1410193323Sed if (JT.empty()) return; 1411199481Srdivacky 1412193323Sed unsigned NumEntries = 0; 1413193323Sed for (unsigned i = 0, e = JT.size(); i != e; ++i) 1414193323Sed NumEntries += JT[i].MBBs.size(); 1415193323Sed 1416203954Srdivacky unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getTargetData()); 1417193323Sed 1418193323Sed // Just allocate space for all the jump tables now. We will fix up the actual 1419193323Sed // MBB entries in the tables after we emit the code for each block, since then 1420193323Sed // we will know the final locations of the MBBs in memory. 1421193323Sed JumpTable = MJTI; 1422203954Srdivacky JumpTableBase = allocateSpace(NumEntries * EntrySize, 1423203954Srdivacky MJTI->getEntryAlignment(*TheJIT->getTargetData())); 1424193323Sed} 1425193323Sed 1426193323Sedvoid JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { 1427193323Sed if (TheJIT->getJITInfo().hasCustomJumpTables()) 1428193323Sed return; 1429193323Sed 1430193323Sed const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1431193323Sed if (JT.empty() || JumpTableBase == 0) return; 1432199481Srdivacky 1433203954Srdivacky 1434203954Srdivacky switch (MJTI->getEntryKind()) { 1435205218Srdivacky case MachineJumpTableInfo::EK_Inline: 1436205218Srdivacky return; 1437203954Srdivacky case MachineJumpTableInfo::EK_BlockAddress: { 1438203954Srdivacky // EK_BlockAddress - Each entry is a plain address of block, e.g.: 1439203954Srdivacky // .word LBB123 1440203954Srdivacky assert(MJTI->getEntrySize(*TheJIT->getTargetData()) == sizeof(void*) && 1441203954Srdivacky "Cross JIT'ing?"); 1442203954Srdivacky 1443203954Srdivacky // For each jump table, map each target in the jump table to the address of 1444203954Srdivacky // an emitted MachineBasicBlock. 1445203954Srdivacky intptr_t *SlotPtr = (intptr_t*)JumpTableBase; 1446203954Srdivacky 1447203954Srdivacky for (unsigned i = 0, e = JT.size(); i != e; ++i) { 1448203954Srdivacky const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 1449203954Srdivacky // Store the address of the basic block for this jump table slot in the 1450203954Srdivacky // memory we allocated for the jump table in 'initJumpTableInfo' 1451203954Srdivacky for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) 1452203954Srdivacky *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); 1453203954Srdivacky } 1454203954Srdivacky break; 1455203954Srdivacky } 1456203954Srdivacky 1457203954Srdivacky case MachineJumpTableInfo::EK_Custom32: 1458203954Srdivacky case MachineJumpTableInfo::EK_GPRel32BlockAddress: 1459203954Srdivacky case MachineJumpTableInfo::EK_LabelDifference32: { 1460203954Srdivacky assert(MJTI->getEntrySize(*TheJIT->getTargetData()) == 4&&"Cross JIT'ing?"); 1461193323Sed // For each jump table, place the offset from the beginning of the table 1462193323Sed // to the target address. 1463193323Sed int *SlotPtr = (int*)JumpTableBase; 1464193323Sed 1465193323Sed for (unsigned i = 0, e = JT.size(); i != e; ++i) { 1466193323Sed const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 1467193323Sed // Store the offset of the basic block for this jump table slot in the 1468193323Sed // memory we allocated for the jump table in 'initJumpTableInfo' 1469193323Sed uintptr_t Base = (uintptr_t)SlotPtr; 1470193323Sed for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { 1471193323Sed uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); 1472203954Srdivacky /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook. 1473193323Sed *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); 1474193323Sed } 1475193323Sed } 1476203954Srdivacky break; 1477193323Sed } 1478203954Srdivacky } 1479193323Sed} 1480193323Sed 1481201360Srdivackyvoid JITEmitter::startGVStub(const GlobalValue* GV, 1482199989Srdivacky unsigned StubSize, unsigned Alignment) { 1483201360Srdivacky SavedBufferBegin = BufferBegin; 1484201360Srdivacky SavedBufferEnd = BufferEnd; 1485201360Srdivacky SavedCurBufferPtr = CurBufferPtr; 1486199481Srdivacky 1487193323Sed BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); 1488193323Sed BufferEnd = BufferBegin+StubSize+1; 1489193323Sed} 1490193323Sed 1491201360Srdivackyvoid JITEmitter::startGVStub(void *Buffer, unsigned StubSize) { 1492201360Srdivacky SavedBufferBegin = BufferBegin; 1493201360Srdivacky SavedBufferEnd = BufferEnd; 1494201360Srdivacky SavedCurBufferPtr = CurBufferPtr; 1495199481Srdivacky 1496193574Sed BufferBegin = CurBufferPtr = (uint8_t *)Buffer; 1497193323Sed BufferEnd = BufferBegin+StubSize+1; 1498193323Sed} 1499193323Sed 1500201360Srdivackyvoid JITEmitter::finishGVStub() { 1501199989Srdivacky assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space."); 1502193323Sed NumBytes += getCurrentPCOffset(); 1503201360Srdivacky BufferBegin = SavedBufferBegin; 1504201360Srdivacky BufferEnd = SavedBufferEnd; 1505201360Srdivacky CurBufferPtr = SavedCurBufferPtr; 1506193323Sed} 1507193323Sed 1508201360Srdivackyvoid *JITEmitter::allocIndirectGV(const GlobalValue *GV, 1509201360Srdivacky const uint8_t *Buffer, size_t Size, 1510201360Srdivacky unsigned Alignment) { 1511201360Srdivacky uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment); 1512201360Srdivacky memcpy(IndGV, Buffer, Size); 1513201360Srdivacky return IndGV; 1514201360Srdivacky} 1515201360Srdivacky 1516193323Sed// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry 1517193323Sed// in the constant pool that was last emitted with the 'emitConstantPool' 1518193323Sed// method. 1519193323Sed// 1520193323Seduintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { 1521193323Sed assert(ConstantNum < ConstantPool->getConstants().size() && 1522193323Sed "Invalid ConstantPoolIndex!"); 1523193323Sed return ConstPoolAddresses[ConstantNum]; 1524193323Sed} 1525193323Sed 1526193323Sed// getJumpTableEntryAddress - Return the address of the JumpTable with index 1527193323Sed// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' 1528193323Sed// 1529193323Seduintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { 1530193323Sed const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); 1531193323Sed assert(Index < JT.size() && "Invalid jump table index!"); 1532199481Srdivacky 1533203954Srdivacky unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getTargetData()); 1534203954Srdivacky 1535193323Sed unsigned Offset = 0; 1536193323Sed for (unsigned i = 0; i < Index; ++i) 1537193323Sed Offset += JT[i].MBBs.size(); 1538199481Srdivacky 1539193323Sed Offset *= EntrySize; 1540199481Srdivacky 1541193323Sed return (uintptr_t)((char *)JumpTableBase + Offset); 1542193323Sed} 1543193323Sed 1544198892Srdivackyvoid JITEmitter::EmittedFunctionConfig::onDelete( 1545198892Srdivacky JITEmitter *Emitter, const Function *F) { 1546198892Srdivacky Emitter->deallocateMemForFunction(F); 1547198892Srdivacky} 1548198892Srdivackyvoid JITEmitter::EmittedFunctionConfig::onRAUW( 1549198892Srdivacky JITEmitter *, const Function*, const Function*) { 1550198892Srdivacky llvm_unreachable("The JIT doesn't know how to handle a" 1551198892Srdivacky " RAUW on a value it has emitted."); 1552198892Srdivacky} 1553198892Srdivacky 1554198892Srdivacky 1555193323Sed//===----------------------------------------------------------------------===// 1556193323Sed// Public interface to this file 1557193323Sed//===----------------------------------------------------------------------===// 1558193323Sed 1559198090SrdivackyJITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM, 1560198090Srdivacky TargetMachine &tm) { 1561198090Srdivacky return new JITEmitter(jit, JMM, tm); 1562193323Sed} 1563193323Sed 1564193323Sed// getPointerToFunctionOrStub - If the specified function has been 1565193323Sed// code-gen'd, return a pointer to the function. If not, compile it, or use 1566193323Sed// a stub to implement lazy compilation if available. 1567193323Sed// 1568193323Sedvoid *JIT::getPointerToFunctionOrStub(Function *F) { 1569193323Sed // If we have already code generated the function, just return the address. 1570193323Sed if (void *Addr = getPointerToGlobalIfAvailable(F)) 1571193323Sed return Addr; 1572199481Srdivacky 1573193323Sed // Get a stub if the target supports it. 1574193323Sed assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1575193323Sed JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 1576199989Srdivacky return JE->getJITResolver().getLazyFunctionStub(F); 1577193323Sed} 1578193323Sed 1579193323Sedvoid JIT::updateFunctionStub(Function *F) { 1580193323Sed // Get the empty stub we generated earlier. 1581193323Sed assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1582193323Sed JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 1583199989Srdivacky void *Stub = JE->getJITResolver().getLazyFunctionStub(F); 1584199989Srdivacky void *Addr = getPointerToGlobalIfAvailable(F); 1585201360Srdivacky assert(Addr != Stub && "Function must have non-stub address to be updated."); 1586193323Sed 1587193323Sed // Tell the target jit info to rewrite the stub at the specified address, 1588193323Sed // rather than creating a new one. 1589199989Srdivacky TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout(); 1590201360Srdivacky JE->startGVStub(Stub, layout.Size); 1591199989Srdivacky getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter()); 1592201360Srdivacky JE->finishGVStub(); 1593193323Sed} 1594193323Sed 1595193323Sed/// freeMachineCodeForFunction - release machine code memory for given Function. 1596193323Sed/// 1597193323Sedvoid JIT::freeMachineCodeForFunction(Function *F) { 1598193323Sed // Delete translation for this from the ExecutionEngine, so it will get 1599193323Sed // retranslated next time it is used. 1600198892Srdivacky updateGlobalMapping(F, 0); 1601193323Sed 1602193323Sed // Free the actual memory for the function body and related stuff. 1603193323Sed assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1604193323Sed cast<JITEmitter>(JCE)->deallocateMemForFunction(F); 1605193323Sed} 1606