1//===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// Contains utilities for adding indirections and breaking up modules. 10// 11//===----------------------------------------------------------------------===// 12 13#ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H 14#define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H 15 16#include "llvm/ADT/StringMap.h" 17#include "llvm/ADT/StringRef.h" 18#include "llvm/ADT/Twine.h" 19#include "llvm/ExecutionEngine/JITSymbol.h" 20#include "llvm/ExecutionEngine/Orc/Core.h" 21#include "llvm/Support/Error.h" 22#include "llvm/Support/Memory.h" 23#include "llvm/Support/Process.h" 24#include "llvm/Transforms/Utils/ValueMapper.h" 25#include <algorithm> 26#include <cassert> 27#include <cstdint> 28#include <functional> 29#include <map> 30#include <memory> 31#include <system_error> 32#include <utility> 33#include <vector> 34 35namespace llvm { 36 37class Constant; 38class Function; 39class FunctionType; 40class GlobalAlias; 41class GlobalVariable; 42class Module; 43class PointerType; 44class Triple; 45class Value; 46 47namespace orc { 48 49/// Base class for pools of compiler re-entry trampolines. 50/// These trampolines are callable addresses that save all register state 51/// before calling a supplied function to return the trampoline landing 52/// address, then restore all state before jumping to that address. They 53/// are used by various ORC APIs to support lazy compilation 54class TrampolinePool { 55public: 56 virtual ~TrampolinePool() {} 57 58 /// Get an available trampoline address. 59 /// Returns an error if no trampoline can be created. 60 virtual Expected<JITTargetAddress> getTrampoline() = 0; 61 62private: 63 virtual void anchor(); 64}; 65 66/// A trampoline pool for trampolines within the current process. 67template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool { 68public: 69 using GetTrampolineLandingFunction = 70 std::function<JITTargetAddress(JITTargetAddress TrampolineAddr)>; 71 72 /// Creates a LocalTrampolinePool with the given RunCallback function. 73 /// Returns an error if this function is unable to correctly allocate, write 74 /// and protect the resolver code block. 75 static Expected<std::unique_ptr<LocalTrampolinePool>> 76 Create(GetTrampolineLandingFunction GetTrampolineLanding) { 77 Error Err = Error::success(); 78 79 auto LTP = std::unique_ptr<LocalTrampolinePool>( 80 new LocalTrampolinePool(std::move(GetTrampolineLanding), Err)); 81 82 if (Err) 83 return std::move(Err); 84 return std::move(LTP); 85 } 86 87 /// Get a free trampoline. Returns an error if one can not be provided (e.g. 88 /// because the pool is empty and can not be grown). 89 Expected<JITTargetAddress> getTrampoline() override { 90 std::lock_guard<std::mutex> Lock(LTPMutex); 91 if (AvailableTrampolines.empty()) { 92 if (auto Err = grow()) 93 return std::move(Err); 94 } 95 assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool"); 96 auto TrampolineAddr = AvailableTrampolines.back(); 97 AvailableTrampolines.pop_back(); 98 return TrampolineAddr; 99 } 100 101 /// Returns the given trampoline to the pool for re-use. 102 void releaseTrampoline(JITTargetAddress TrampolineAddr) { 103 std::lock_guard<std::mutex> Lock(LTPMutex); 104 AvailableTrampolines.push_back(TrampolineAddr); 105 } 106 107private: 108 static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) { 109 LocalTrampolinePool<ORCABI> *TrampolinePool = 110 static_cast<LocalTrampolinePool *>(TrampolinePoolPtr); 111 return TrampolinePool->GetTrampolineLanding(static_cast<JITTargetAddress>( 112 reinterpret_cast<uintptr_t>(TrampolineId))); 113 } 114 115 LocalTrampolinePool(GetTrampolineLandingFunction GetTrampolineLanding, 116 Error &Err) 117 : GetTrampolineLanding(std::move(GetTrampolineLanding)) { 118 119 ErrorAsOutParameter _(&Err); 120 121 /// Try to set up the resolver block. 122 std::error_code EC; 123 ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 124 ORCABI::ResolverCodeSize, nullptr, 125 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); 126 if (EC) { 127 Err = errorCodeToError(EC); 128 return; 129 } 130 131 ORCABI::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()), 132 &reenter, this); 133 134 EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(), 135 sys::Memory::MF_READ | 136 sys::Memory::MF_EXEC); 137 if (EC) { 138 Err = errorCodeToError(EC); 139 return; 140 } 141 } 142 143 Error grow() { 144 assert(this->AvailableTrampolines.empty() && "Growing prematurely?"); 145 146 std::error_code EC; 147 auto TrampolineBlock = 148 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 149 sys::Process::getPageSizeEstimate(), nullptr, 150 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); 151 if (EC) 152 return errorCodeToError(EC); 153 154 unsigned NumTrampolines = 155 (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) / 156 ORCABI::TrampolineSize; 157 158 uint8_t *TrampolineMem = static_cast<uint8_t *>(TrampolineBlock.base()); 159 ORCABI::writeTrampolines(TrampolineMem, ResolverBlock.base(), 160 NumTrampolines); 161 162 for (unsigned I = 0; I < NumTrampolines; ++I) 163 this->AvailableTrampolines.push_back( 164 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>( 165 TrampolineMem + (I * ORCABI::TrampolineSize)))); 166 167 if (auto EC = sys::Memory::protectMappedMemory( 168 TrampolineBlock.getMemoryBlock(), 169 sys::Memory::MF_READ | sys::Memory::MF_EXEC)) 170 return errorCodeToError(EC); 171 172 TrampolineBlocks.push_back(std::move(TrampolineBlock)); 173 return Error::success(); 174 } 175 176 GetTrampolineLandingFunction GetTrampolineLanding; 177 178 std::mutex LTPMutex; 179 sys::OwningMemoryBlock ResolverBlock; 180 std::vector<sys::OwningMemoryBlock> TrampolineBlocks; 181 std::vector<JITTargetAddress> AvailableTrampolines; 182}; 183 184/// Target-independent base class for compile callback management. 185class JITCompileCallbackManager { 186public: 187 using CompileFunction = std::function<JITTargetAddress()>; 188 189 virtual ~JITCompileCallbackManager() = default; 190 191 /// Reserve a compile callback. 192 Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile); 193 194 /// Execute the callback for the given trampoline id. Called by the JIT 195 /// to compile functions on demand. 196 JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr); 197 198protected: 199 /// Construct a JITCompileCallbackManager. 200 JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP, 201 ExecutionSession &ES, 202 JITTargetAddress ErrorHandlerAddress) 203 : TP(std::move(TP)), ES(ES), 204 CallbacksJD(ES.createJITDylib("<Callbacks>")), 205 ErrorHandlerAddress(ErrorHandlerAddress) {} 206 207 void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) { 208 this->TP = std::move(TP); 209 } 210 211private: 212 std::mutex CCMgrMutex; 213 std::unique_ptr<TrampolinePool> TP; 214 ExecutionSession &ES; 215 JITDylib &CallbacksJD; 216 JITTargetAddress ErrorHandlerAddress; 217 std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol; 218 size_t NextCallbackId = 0; 219}; 220 221/// Manage compile callbacks for in-process JITs. 222template <typename ORCABI> 223class LocalJITCompileCallbackManager : public JITCompileCallbackManager { 224public: 225 /// Create a new LocalJITCompileCallbackManager. 226 static Expected<std::unique_ptr<LocalJITCompileCallbackManager>> 227 Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) { 228 Error Err = Error::success(); 229 auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>( 230 new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err)); 231 if (Err) 232 return std::move(Err); 233 return std::move(CCMgr); 234 } 235 236private: 237 /// Construct a InProcessJITCompileCallbackManager. 238 /// @param ErrorHandlerAddress The address of an error handler in the target 239 /// process to be used if a compile callback fails. 240 LocalJITCompileCallbackManager(ExecutionSession &ES, 241 JITTargetAddress ErrorHandlerAddress, 242 Error &Err) 243 : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) { 244 ErrorAsOutParameter _(&Err); 245 auto TP = LocalTrampolinePool<ORCABI>::Create( 246 [this](JITTargetAddress TrampolineAddr) { 247 return executeCompileCallback(TrampolineAddr); 248 }); 249 250 if (!TP) { 251 Err = TP.takeError(); 252 return; 253 } 254 255 setTrampolinePool(std::move(*TP)); 256 } 257}; 258 259/// Base class for managing collections of named indirect stubs. 260class IndirectStubsManager { 261public: 262 /// Map type for initializing the manager. See init. 263 using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>; 264 265 virtual ~IndirectStubsManager() = default; 266 267 /// Create a single stub with the given name, target address and flags. 268 virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr, 269 JITSymbolFlags StubFlags) = 0; 270 271 /// Create StubInits.size() stubs with the given names, target 272 /// addresses, and flags. 273 virtual Error createStubs(const StubInitsMap &StubInits) = 0; 274 275 /// Find the stub with the given name. If ExportedStubsOnly is true, 276 /// this will only return a result if the stub's flags indicate that it 277 /// is exported. 278 virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0; 279 280 /// Find the implementation-pointer for the stub. 281 virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0; 282 283 /// Change the value of the implementation pointer for the stub. 284 virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0; 285 286private: 287 virtual void anchor(); 288}; 289 290/// IndirectStubsManager implementation for the host architecture, e.g. 291/// OrcX86_64. (See OrcArchitectureSupport.h). 292template <typename TargetT> 293class LocalIndirectStubsManager : public IndirectStubsManager { 294public: 295 Error createStub(StringRef StubName, JITTargetAddress StubAddr, 296 JITSymbolFlags StubFlags) override { 297 std::lock_guard<std::mutex> Lock(StubsMutex); 298 if (auto Err = reserveStubs(1)) 299 return Err; 300 301 createStubInternal(StubName, StubAddr, StubFlags); 302 303 return Error::success(); 304 } 305 306 Error createStubs(const StubInitsMap &StubInits) override { 307 std::lock_guard<std::mutex> Lock(StubsMutex); 308 if (auto Err = reserveStubs(StubInits.size())) 309 return Err; 310 311 for (auto &Entry : StubInits) 312 createStubInternal(Entry.first(), Entry.second.first, 313 Entry.second.second); 314 315 return Error::success(); 316 } 317 318 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override { 319 std::lock_guard<std::mutex> Lock(StubsMutex); 320 auto I = StubIndexes.find(Name); 321 if (I == StubIndexes.end()) 322 return nullptr; 323 auto Key = I->second.first; 324 void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second); 325 assert(StubAddr && "Missing stub address"); 326 auto StubTargetAddr = 327 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr)); 328 auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second); 329 if (ExportedStubsOnly && !StubSymbol.getFlags().isExported()) 330 return nullptr; 331 return StubSymbol; 332 } 333 334 JITEvaluatedSymbol findPointer(StringRef Name) override { 335 std::lock_guard<std::mutex> Lock(StubsMutex); 336 auto I = StubIndexes.find(Name); 337 if (I == StubIndexes.end()) 338 return nullptr; 339 auto Key = I->second.first; 340 void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second); 341 assert(PtrAddr && "Missing pointer address"); 342 auto PtrTargetAddr = 343 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr)); 344 return JITEvaluatedSymbol(PtrTargetAddr, I->second.second); 345 } 346 347 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override { 348 using AtomicIntPtr = std::atomic<uintptr_t>; 349 350 std::lock_guard<std::mutex> Lock(StubsMutex); 351 auto I = StubIndexes.find(Name); 352 assert(I != StubIndexes.end() && "No stub pointer for symbol"); 353 auto Key = I->second.first; 354 AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>( 355 IndirectStubsInfos[Key.first].getPtr(Key.second)); 356 *AtomicStubPtr = static_cast<uintptr_t>(NewAddr); 357 return Error::success(); 358 } 359 360private: 361 Error reserveStubs(unsigned NumStubs) { 362 if (NumStubs <= FreeStubs.size()) 363 return Error::success(); 364 365 unsigned NewStubsRequired = NumStubs - FreeStubs.size(); 366 unsigned NewBlockId = IndirectStubsInfos.size(); 367 typename TargetT::IndirectStubsInfo ISI; 368 if (auto Err = 369 TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired, nullptr)) 370 return Err; 371 for (unsigned I = 0; I < ISI.getNumStubs(); ++I) 372 FreeStubs.push_back(std::make_pair(NewBlockId, I)); 373 IndirectStubsInfos.push_back(std::move(ISI)); 374 return Error::success(); 375 } 376 377 void createStubInternal(StringRef StubName, JITTargetAddress InitAddr, 378 JITSymbolFlags StubFlags) { 379 auto Key = FreeStubs.back(); 380 FreeStubs.pop_back(); 381 *IndirectStubsInfos[Key.first].getPtr(Key.second) = 382 reinterpret_cast<void *>(static_cast<uintptr_t>(InitAddr)); 383 StubIndexes[StubName] = std::make_pair(Key, StubFlags); 384 } 385 386 std::mutex StubsMutex; 387 std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos; 388 using StubKey = std::pair<uint16_t, uint16_t>; 389 std::vector<StubKey> FreeStubs; 390 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; 391}; 392 393/// Create a local compile callback manager. 394/// 395/// The given target triple will determine the ABI, and the given 396/// ErrorHandlerAddress will be used by the resulting compile callback 397/// manager if a compile callback fails. 398Expected<std::unique_ptr<JITCompileCallbackManager>> 399createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, 400 JITTargetAddress ErrorHandlerAddress); 401 402/// Create a local indriect stubs manager builder. 403/// 404/// The given target triple will determine the ABI. 405std::function<std::unique_ptr<IndirectStubsManager>()> 406createLocalIndirectStubsManagerBuilder(const Triple &T); 407 408/// Build a function pointer of FunctionType with the given constant 409/// address. 410/// 411/// Usage example: Turn a trampoline address into a function pointer constant 412/// for use in a stub. 413Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr); 414 415/// Create a function pointer with the given type, name, and initializer 416/// in the given Module. 417GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name, 418 Constant *Initializer); 419 420/// Turn a function declaration into a stub function that makes an 421/// indirect call using the given function pointer. 422void makeStub(Function &F, Value &ImplPointer); 423 424/// Promotes private symbols to global hidden, and renames to prevent clashes 425/// with other promoted symbols. The same SymbolPromoter instance should be 426/// used for all symbols to be added to a single JITDylib. 427class SymbolLinkagePromoter { 428public: 429 /// Promote symbols in the given module. Returns the set of global values 430 /// that have been renamed/promoted. 431 std::vector<GlobalValue *> operator()(Module &M); 432 433private: 434 unsigned NextId = 0; 435}; 436 437/// Clone a function declaration into a new module. 438/// 439/// This function can be used as the first step towards creating a callback 440/// stub (see makeStub), or moving a function body (see moveFunctionBody). 441/// 442/// If the VMap argument is non-null, a mapping will be added between F and 443/// the new declaration, and between each of F's arguments and the new 444/// declaration's arguments. This map can then be passed in to moveFunction to 445/// move the function body if required. Note: When moving functions between 446/// modules with these utilities, all decls should be cloned (and added to a 447/// single VMap) before any bodies are moved. This will ensure that references 448/// between functions all refer to the versions in the new module. 449Function *cloneFunctionDecl(Module &Dst, const Function &F, 450 ValueToValueMapTy *VMap = nullptr); 451 452/// Move the body of function 'F' to a cloned function declaration in a 453/// different module (See related cloneFunctionDecl). 454/// 455/// If the target function declaration is not supplied via the NewF parameter 456/// then it will be looked up via the VMap. 457/// 458/// This will delete the body of function 'F' from its original parent module, 459/// but leave its declaration. 460void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, 461 ValueMaterializer *Materializer = nullptr, 462 Function *NewF = nullptr); 463 464/// Clone a global variable declaration into a new module. 465GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, 466 ValueToValueMapTy *VMap = nullptr); 467 468/// Move global variable GV from its parent module to cloned global 469/// declaration in a different module. 470/// 471/// If the target global declaration is not supplied via the NewGV parameter 472/// then it will be looked up via the VMap. 473/// 474/// This will delete the initializer of GV from its original parent module, 475/// but leave its declaration. 476void moveGlobalVariableInitializer(GlobalVariable &OrigGV, 477 ValueToValueMapTy &VMap, 478 ValueMaterializer *Materializer = nullptr, 479 GlobalVariable *NewGV = nullptr); 480 481/// Clone a global alias declaration into a new module. 482GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, 483 ValueToValueMapTy &VMap); 484 485/// Clone module flags metadata into the destination module. 486void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, 487 ValueToValueMapTy &VMap); 488 489} // end namespace orc 490 491} // end namespace llvm 492 493#endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H 494